diff --git a/.coveragerc b/.coveragerc index 32deb76b5..846588761 100644 --- a/.coveragerc +++ b/.coveragerc @@ -5,7 +5,6 @@ omit = *__init__* */scripts/* */setup.py - *rllab* */flow/utils/shflags *dummy_server.py *aimsun.py @@ -13,9 +12,14 @@ omit = *aimsun/api.py *aimsun/generate.py *aimsun/load.py + *flow/multiagent_* + *flow/singleagent_* exclude_lines = if __name__ == .__main__.: raise NotImplementedError @ray.remote + @abstractmethod def policy_mapping_fn* + def main(args)* + pragma: no cover diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md index bbc2ee05e..1cb6b448d 100644 --- a/.github/ISSUE_TEMPLATE/question.md +++ b/.github/ISSUE_TEMPLATE/question.md @@ -8,7 +8,10 @@ about: If you have a question or run into some problem with Flow (e.g. during th PLEASE DO NOT ASK YOUR QUESTIONS HERE ON GITHUB! If you have a question or run into some problem (e.g. during installation or when running an example) -with Flow, please direct your technical questions to Stack Overflow using the "flow-project" tag. +with Flow, please direct technical questions to the Flow project Slack channel here: -link: https://stackoverflow.com/questions/tagged/flow-project -tag: flow-project +https://join.slack.com/t/flow-users/shared_invite/enQtODQ0NDYxMTQyNDY2LTY1ZDVjZTljM2U0ODIxNTY5NTQ2MmUxMzYzNzc5NzU4ZTlmNGI2ZjFmNGU4YjVhNzE3NjcwZTBjNzIxYTg5ZmY + +If you have a non-technical inquiry, please send us an email: + +https://flow-project.github.io/contact.html diff --git a/.travis.yml b/.travis.yml index 30f3174a4..297281bc7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -54,6 +54,17 @@ before_install: - export AIMSUN_SITEPACKAGES="/home/travis/miniconda/envs/aimsun_flow" - export AIMSUN_NEXT_PATH="/home/user/Aimsun_Next_XXX" + # Install stable-baselines + - pip install stable_baselines==2.7.0 + + # Install h-baselines + - pushd $HOME + - git clone https://github.com/AboudyKreidieh/h-baselines.git + - pushd h-baselines + - pip install -e . + - popd + - popd + - ls ../ install: @@ -62,7 +73,6 @@ install: - pip install -e . - pip install coveralls - pip install jupyter - - pip install stable_baselines==2.7.0 script: - nose2 --with-coverage diff --git a/CODEOWNERS b/CODEOWNERS index 175783157..1a0ba0d97 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -2,7 +2,7 @@ # Each line is a file pattern followed by one or more owners. # These owners will be the default owners for everything in the repo. -* @cathywu @eugenevinitsky @AboudyKreidieh @kanaadp +* @eugenevinitsky @AboudyKreidieh @kjang96 # Order is important. The last matching pattern has the most precedence. # So if a pull request only touches javascript files, only these owners diff --git a/README.md b/README.md index 7170f47d2..7d37223c5 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ See [our website](https://flow-project.github.io/) for more information on the a # Technical questions -Please direct your technical questions to [Stack Overflow](https://stackoverflow.com) using the [flow-project](https://stackoverflow.com/questions/tagged/flow-project) tag. +If you have a bug, please report it. Otherwise, join the [Flow Users group](https://join.slack.com/t/flow-users/shared_invite/enQtODQ0NDYxMTQyNDY2LTY1ZDVjZTljM2U0ODIxNTY5NTQ2MmUxMzYzNzc5NzU4ZTlmNGI2ZjFmNGU4YjVhNzE3NjcwZTBjNzIxYTg5ZmY) on Slack! # Getting involved diff --git a/docs/libsumo_mac.md b/docs/libsumo_mac.md new file mode 100644 index 000000000..7a02d3823 --- /dev/null +++ b/docs/libsumo_mac.md @@ -0,0 +1,23 @@ +# How to install Libsumo for Mac OS + +This is adapted from an email exchange with the SUMO staff. + + + +To install libsumo requires re-building and installing SUMO from source. + +## Steps + +- **Install swig:** ```brew install swig``` +- **Clone the repo:** ```git clone https://github.com/eclipse/sumo.git``` +- **Create a “cmake-build” directory inside sumo/build/ and navigate to it:** ```mkdir build/cmake-build && cd build/cmake-build``` + +**The next 3 steps are inside that directory** + +- ```cmake ../..``` +- ```make``` +- ```make install``` + +## Additional Notes +- You can test if libsumo has been built looking at (./testlibsumo) inside the sumo/bin/ directory. +- Bear in mind to use libsumo with the same Python version with which CMake built SUMO. \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index d228c6095..b2dc27e75 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -32,9 +32,6 @@ def __getattr__(cls, name): MOCK_MODULES = [ - 'rllab', - "rllab.core.serializable.Serializable", - 'rllab.sampler.utils.rollout', "AAPI", 'PyANGKernel', 'AAPI.GKGUISystem', diff --git a/docs/source/examples.rst b/docs/source/examples.rst index ef04f53cd..74bb08ba1 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -9,7 +9,7 @@ A few points of clarification: * Occasionally we describe environments as *fully observed*. By this we mean that all relevant pieces of information (speed, relative positions, traffic light states) of the system are available to the controller. *Partially observed* refers to only a subset being available. This is the default configuration of the environment and the set of observations can be customized. -* Each environment has customizable environment/network parameters that can be used to configure it beyond what is described here. Particularly pertinent parameters are described. Additional parameters can be found by examining the documentation in the relevant scenario and environment files. +* Each environment has customizable environment/network parameters that can be used to configure it beyond what is described here. Particularly pertinent parameters are described. Additional parameters can be found by examining the documentation in the relevant network and environment files. * In the figures below, the following key is used for vehicle colors, where AV stands for autonomous vehicle. @@ -22,12 +22,12 @@ Figure Eight The figure-eight is a closed-ring version of an intersection. The goal is to maximize the system-wide velocity for fourteen vehicles, which necessitates spacing the vehicles so that they don't -run into conflicts at the merging points. The scenario is fully observed: all vehicles +run into conflicts at the merging points. The network is fully observed: all vehicles speeds and positions are visible to the controller. -This scenario is also a benchmark, and has been +This network is also a benchmark, and has been extensively tested at three penetration rates: 1 AV 13 humans, 7 AVs 7 humans, 14 AVs. -The scenario, pictured below, +The network, pictured below, is relatively light-weight and can be trained the quickest. It can serve both as a test that the training process is working correctly and as a study of the difficulty of controlling many vehicles simultaneously. @@ -43,15 +43,14 @@ In this scenario, human drivers placed on a ring develop a travelling wave that decelerations and lowers the average velocity of the system. The goal is to train a single autonomous vehicle to eliminate the shockwave. -The pre-built run script for running this in human-only mode is `examples/sumo/sugiyama.py`. -The run scripts in rllib and aimsun are `examples/rllib/stabilizing_the_ring.py` and -`examples/aimsun/stabilizing_the_ring.py`. These control environments are partially observed: -the autonomous vehicle only observes its own distance to the leading vehicle, its speed, -and the speed of the leading vehicle. +The experiment config for this in human-only mode (non-RL) is `examples/exp_configs/non_rl/ring.py`. +The experiment config for RL-based is `examples/exp_configs/rl/singleagent/singleagent_ring.py`. +These control environments are partially observed: the autonomous vehicle only observes its own +distance to the leading vehicle, its speed, and the speed of the leading vehicle. To make this task more difficult, the environment has a configurable parameter, `ring_length`, which can be set to a list containing the minimum and maximum ring-size. The autonomous vehicle must -learn to distinguish these scenarios from each other and pick the appropriate driving behavior. +learn to distinguish these networks from each other and pick the appropriate driving behavior. .. image:: ../img/stabilizing_the_ring.png :width: 400 diff --git a/docs/source/flow_setup.rst b/docs/source/flow_setup.rst index 69884ce57..cbe585d36 100644 --- a/docs/source/flow_setup.rst +++ b/docs/source/flow_setup.rst @@ -2,7 +2,7 @@ .. contents:: Table of contents Local Installation of Flow -================== +========================== To get Flow running, you need three things: Flow, @@ -108,10 +108,28 @@ Note that, if the above commands did not work, you may need to run ``source ~/.bashrc`` or open a new terminal to update your $PATH variable. *Troubleshooting*: -If you are a Mac user and the above command gives you the error ``FXApp:openDisplay: unable to open display :0.0``, make sure to open the application XQuartz. +If you are a Mac user and the above command gives you the error +``FXApp:openDisplay: unable to open display :0.0``, make sure to open the +application XQuartz. -Testing your installation -~~~~~~~~~~~~~~~~~~~~~~~~~ +*Troubleshooting*: +If you are a Mac user and the above command gives you the error +``Segmentation fault: 11``, make sure to reinstall ``fox`` using brew. +:: + + # Uninstall Catalina bottle of fox: + $ brew uninstall --ignore-dependencies fox + + # Edit brew Formula of fox: + $ brew edit fox + + # Comment out or delete the following line: sha256 "c6697be294c9a0458580564d59f8db32791beb5e67a05a6246e0b969ffc068bc" => :catalina + # Install Mojave bottle of fox: + $ brew install fox + + +Testing your SUMO and Flow installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Once the above modules have been successfully installed, we can test the installation by running a few examples. Before trying to run any examples, be @@ -124,17 +142,17 @@ sure to enter your conda environment by typing: Let’s see some traffic action: :: - - python examples/sumo/sugiyama.py + + python examples/simulate.py ring Running the following should result in the loading of the SUMO GUI. Click the run button and you should see unstable traffic form after a few seconds, a la (Sugiyama et al, 2008). This means that you have Flow -properly configured with SUMO and Flow! +properly configured with SUMO! (Optional) Installing Aimsun ------------------ +---------------------------- In addition to SUMO, Flow supports the use of the traffic simulator "Aimsun". In order setup Flow with Aimsun, you will first need to install Aimsun. This @@ -151,7 +169,10 @@ during the execution of various tasks. The path should look something like: export AIMSUN_NEXT_PATH="/home/user/Aimsun_Next_X_Y_Z/" # Linux export AIMSUN_NEXT_PATH="/Applications/Aimsun Next.app/Contents/MacOS/" # OS X -`Note for Mac users:` when you download Aimsun, you will get a folder named "Programming". You need to rename it to "programming" (all lowercase) and to move it inside the "Aimsun Next.app/Contents/MacOS/" directory so that the python API can work. +`Note for Mac users:` when you download Aimsun, you will get a folder named +"Programming". You need to rename it to "programming" (all lowercase) and to +move it inside the "Aimsun Next.app/Contents/MacOS/" directory so that the +python API can work. In addition, being that Aimsun's python API is written to support Python 2.7.4, we will need to create a Python 2.7.4 conda environment that Aimsun can refer @@ -178,15 +199,15 @@ The latter command should return an output similar to: /path/to/envs/aimsun_flow/bin/python -Copy the path up until right before /bin (i.e. /path/to/envs/aimsun_flow) and +Copy the path up until right before /lib (i.e. /path/to/envs/aimsun_flow) and place it under the `AIMSUN_SITEPACKAGES` variable in your bashrc, like this: :: - export AIMSUN_SITEPACKAGES="/path/to/envs/aimsun_flow/bin/python" + export AIMSUN_SITEPACKAGES="/path/to/envs/aimsun_flow" -Testing your installation -~~~~~~~~~~~~~~~~~~~~~~~~~ +Testing your Aimsun installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To test that you installation was successful, you can try running one of the Aimsun examples within the Flow main directory. In order to do so, you need @@ -196,8 +217,13 @@ to activate the `flow` env. Type: source deactivate aimsun_flow source activate flow - python examples/aimsun/sugiyama.py + python examples/simulate.py ring --aimsun +*Troubleshootig for Ubuntu users with Aimsun 8.4*: when you run the above +example, you may get a subprocess.Popen error ``OSError: [Errno 8] Exec format error:``. +To fix this, go to the `Aimsun Next` main directory, open the `Aimsun_Next` +binary with a text editor and add the shebang to the first line of the script +``#!/bin/sh``. (Optional) Install Ray RLlib ---------------------------- @@ -209,10 +235,11 @@ RLlib is one such library. First visit and install the required packages. -If you are not intending to develop RL algorithms or customize rllib you don't need to do anything, -Ray was installed when you created the conda environment. +If you are not intending to develop RL algorithms or customize rllib you don't +need to do anything, Ray was installed when you created the conda environment. -If you are intending to modify Ray, the installation process for this library is as follows: +If you are intending to modify Ray, the installation process for this library +is as follows: :: @@ -226,22 +253,8 @@ required libraries as specified at and then follow the setup instructions. -(Optional) Install Stable Baselines ----------------------------- - -An additional library that Flow supports is the fork of OpenAI's Baselines, Stable-Baselines. -First visit and -install the required packages and pip install the stable baselines package as described in their -installation instructions. - -You can test your installation by running - -:: - - python examples/stable_baselines/stabilizing_the_ring.py - -Testing your installation -~~~~~~~~~~~~~~~~~~~~~~~~~ +Testing your RLlib installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ See `getting started with RLlib `_ for sample commands. @@ -255,11 +268,16 @@ In order to test run an Flow experiment in RLlib, try the following command: :: - python examples/rllib/stabilizing_the_ring.py + python examples/train.py singleagent_ring + If it does not fail, this means that you have Flow properly configured with RLlib. + +Visualizing with Tensorboard +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + To visualize the training progress: :: @@ -272,9 +290,10 @@ If tensorboard is not installed, you can install with pip: pip install tensorboard -For information on how to deploy a cluster, refer to the `Ray instructions `_. -The basic workflow is running the following locally, ssh-ing into the host machine, and starting -jobs from there. +For information on how to deploy a cluster, refer to the +`Ray instructions `_. +The basic workflow is running the following locally, ssh-ing into the host +machine, and starting jobs from there. :: @@ -283,6 +302,49 @@ jobs from there. ray teardown scripts/ray_autoscale.yaml +(Optional) Install Stable Baselines +----------------------------------- + +An additional library that Flow supports is the fork of OpenAI's Baselines, Stable-Baselines. +First visit and +install the required packages and pip install the stable baselines package as described in their +installation instructions. + +Testing your Stable Baselines installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can test your installation by running + +:: + + python examples/train.py singleagent_ring --rl_trainer Stable-Baselines + + +(Optional) Install h-baselines +------------------------------ + +h-baselines is another variant of stable-baselines that support the use of +single-agent, multiagent, and hierarchical policies. To install h-baselines, +run the following commands: + +:: + + git clone https://github.com/AboudyKreidieh/h-baselines.git + cd h-baselines + source activate flow # if using a Flow environment + pip install -e . + + +Testing your h-baselines installation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +You can test your installation by running + +:: + + python examples/train.py singleagent_ring --rl_trainer h-baselines + + (Optional) Direct install of SUMO from GitHub --------------------------------------------- @@ -345,16 +407,22 @@ If you have Ubuntu 14.04+, run the following command Virtual installation of Flow (using docker containers) -================================ +====================================================== To install a containerized Flow stack, run: + :: + docker run -d -p 5901:5901 -p 6901:6901 fywu85/flow-desktop:latest To access the docker container, go to the following URL and enter the default password `password`: + :: + http://localhost:6901/vnc.html To use the Jupyter Notebook inside the container, run: + :: + jupyter notebook --ip=127.0.0.1 diff --git a/docs/source/index.rst b/docs/source/index.rst index 7a187a0d3..2e1d42a23 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -9,7 +9,7 @@ Welcome to Flow `Flow `_ is a computational framework for deep RL and control experiments for traffic microsimulation. Visit `our website `_ for more information. Flow is a work in progress - input is welcome. Available documentation is limited for now. -`Tutorials `_ are available in iPython notebook format. +`Tutorials `_ are available in iPython notebook format. *If you are looking for Akvo Flow, their documentation can be found at* http://flowsupport.akvo.org. diff --git a/docs/source/multiagent.rst b/docs/source/multiagent.rst index 1e4fa458a..4ce1ce209 100644 --- a/docs/source/multiagent.rst +++ b/docs/source/multiagent.rst @@ -50,4 +50,4 @@ A brief example of a multiagent environment: For further details look at our -`multiagent examples `_. +`multiagent examples `_. diff --git a/environment.yml b/environment.yml index 1eaf3aa3b..97d9ad6f8 100644 --- a/environment.yml +++ b/environment.yml @@ -1,16 +1,17 @@ name: flow dependencies: - - python==3.6.8 - - scipy==1.1.0 - - lxml==4.2.4 - - six==1.11.0 - - path.py - - python-dateutil==2.7.3 - - tensorflow==1.9.0 - - cloudpickle==1.2.1 - - setuptools==41.0.0 + - python==3.7.3 - pip: + - scipy==1.1.0 + - lxml==4.4.1 + - six==1.11.0 + - path.py + - python-dateutil==2.7.3 + - pip>=18.0 + - tensorflow==1.15.2 + - setuptools==41.0.0 + - plotly==2.4.0 - gym==0.14.0 - pyprind==2.11.2 - nose2==0.8.0 @@ -19,9 +20,9 @@ dependencies: - matplotlib==3.0.0 - dill - lz4 - - ray==0.7.3 + - ray==0.8.0 - setproctitle - psutil - opencv-python - - boto3==1.4.8 + - boto3==1.10.45 - redis~=2.10.6 diff --git a/examples/README.md b/examples/README.md index f612bb1fb..a9d681131 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,40 +1,198 @@ # Flow Examples Before continuing to the Flow examples, we recommend **installing Flow** by -executing the following [installation instructions]( +following the [installation instructions]( https://flow.readthedocs.io/en/latest/flow_setup.html). The **examples** folder provides several examples demonstrating how -both simulation and RL-oriented experiments can be setup and executed within -the Flow framework on a variety of traffic problems. These examples are .py -files that may be executed either from terminal or via an editor. For example, -in order to execute the sugiyama example in *examples/sumo*, we run: +both non-RL simulation and RL-oriented simulatons can be setup and executed +within the Flow framework on a variety of traffic problems. These examples are +python files that may be executed either from terminal or via a text editor. +For example, in order to execute the non-RL Ring example we run: -```shell -python /examples/sumo/sugiyama.py +```shell script +python simulate.py "ring" ``` -The examples are distributed into the following sections: +The examples are categorized into the following 3 sections: -**examples/sumo/** contains examples of transportation network with vehicles -following human-dynamical models of driving behavior using the traffic -micro-simulator sumo. +## non-RL examples -**examples/aimsun/** contains examples of transportation network with vehicles +These are examples of transportation network with vehicles following human-dynamical models of driving behavior using the traffic -micro-simulator Aimsun. +micro-simulator sumo and traffic macro-simulator Aimsun. + +To execute these examples, run + +```shell script +python simulate.py EXP_CONFIG +``` +where `EXP_CONFIG` is the name of the experiment configuration file, as located +in `exp_configs/non_rl.` + +There are several *optional* arguments that can be added to the above command: + +```shell script + python simulate.py EXP_CONFIG --num_runs n --no_render --aimsun --gen_emission +``` +where `--num_runs` indicates the number of simulations to run (default of `n` +is 1), `--no_render` indicates whether to deactivate the simulation GUI during +runtime (by default simulation GUI is active), `--aimsun` indicates whether to +run the simulation using the simulator Aimsun (the default simulator is SUMO), +and `--gen_emission` indicates whether to generate an emission file from the +simulation. + +## RL examples + +### RLlib -**examples/rllib/** provides similar networks as those presented in the -previous point, but in the present of autonomous vehicle (AV) or traffic light -agents being trained through RL algorithms provided by *RLlib*. +These examples are similar networks as those mentioned in *non-RL examples*, +but in the presence of autonomous vehicle (AV) or traffic light agents being +trained through RL algorithms provided by *RLlib*. + +To execute these examples, run + +```shell script +python train.py EXP_CONFIG --rl_trainer "rllib" +``` +where `EXP_CONFIG` is the name of the experiment configuration file, as located +in `exp_configs/rl/singleagent` or `exp_configs/rl/multiagent.` + + +### stable-baselines + +These examples provide similar networks as those +mentioned in *non-RL examples*, but in the presence of autonomous vehicle (AV) +or traffic light agents being trained through RL algorithms provided by OpenAI +*stable-baselines*. + +To execute these examples, run + +```shell script +python train.py EXP_CONFIG --rl_trainer "stable-baselines" +``` +where `EXP_CONFIG` is the name of the experiment configuration file, as located +in `exp_configs/rl/singleagent.` + +Note that, currently, multiagent experiments are only supported through RLlib. + +There are several *optional* arguments that can be added to the above command: + +```shell script +python train.py EXP_CONFIG --rl_trainer "stable-baselines" --num_cpus n1 --num_steps n2 --rollout_size r +``` +where `--num_cpus` indicates the number of CPUs to use (default of `n1` is 1), +`--num_steps` indicates the total steps to perform the learning (default of +`n2` is 5000), and `--rollout_size` indicates the number of steps in a training +batch (default of `r` is 1000) + +### h-baselines + +A third RL algorithms package supported by the `train.py` script is +[h-baselines](https://github.com/AboudyKreidieh/h-baselines). In order to use +the algorithms supported by this package, begin by installing h-baselines by +following the setup instructions located +[here](https://flow.readthedocs.io/en/latest/flow_setup.html#optional-install-h-baselines). +A policy can be trained using one of the exp_configs as follows: + +```shell script +python examples/train.py singleagent_ring --rl_trainer h-baselines +``` + +**Logging:** + +The above script executes a training operation and begins logging training and +testing data under the path: *training_data/singleagent_ring/*. + +To visualize the statistics of various tensorflow operations in tensorboard, +type: + +```shell script +tensorboard --logdir /examples/training_data/singleagent_ring/ +``` + +Moreover, as training progressive, per-iteration and cumulative statistics are +printed as a table on your terminal. These statistics are stored under the csv +files *train.csv* and *eval.csv* (if also using an evaluation environment) +within the same directory. + +**Hyperparameters:** + +When using h-baseline, multiple new command-line arguments can be passed to +adjust the choice of algorithm and variable hyperparameters of the algorithms. +These new arguments are as follows: + +* `--alg` (*str*): The algorithm to use. Must be one of [TD3, SAC]. Defaults to + 'TD3'. +* `--evaluate` (*store_true*): whether to add an evaluation environment. The + evaluation environment is similar to the training environment, but with + `env_params.evaluate` set to True. +* `--n_training` (*int*): Number of training operations to perform. Each + training operation is performed on a new seed. Defaults to 1. +* `--total_steps` (*int*): Total number of timesteps used during training. + Defaults to 1000000. +* `--seed` (*int*): Sets the seed for numpy, tensorflow, and random. Defaults + to 1. +* `--log_interval` (*int*): the number of training steps before logging + training results. Defaults to 2000. +* `--eval_interval` (*int*): number of simulation steps in the training + environment before an evaluation is performed. Only relevant if `--evaluate` + is called. Defaults to 50000. +* `--save_interval` (int): number of simulation steps in the training + environment before the model is saved. Defaults to 50000. +* `--initial_exploration_steps` (*int*): number of timesteps that the policy is + run before training to initialize the replay buffer with samples. Defaults to + 10000. +* `--nb_train_steps` (*int*): the number of training steps. Defaults to 1. +* `--nb_rollout_steps` (*int*): the number of rollout steps. Defaults to 1. +* `--nb_eval_episodes` (*int*): the number of evaluation episodes. Only + relevant if `--evaluate` is called. Defaults to 50. +* `--reward_scale` (*float*): the value the reward should be scaled by. + Defaults to 1. +* `--buffer_size` (*int*): the max number of transitions to store. Defaults to + 200000. +* `--batch_size` (*int*): the size of the batch for learning the policy. + Defaults to 128. +* `--actor_lr` (*float*): the actor learning rate. Defaults to 3e-4. +* `--critic_lr` (*float*): the critic learning rate. Defaults to 3e-4. +* `--tau` (*float*): the soft update coefficient (keep old values, between 0 + and 1). Defatuls to 0.005. +* `--gamma` (*float*): the discount rate. Defaults to 0.99. +* `--layer_norm` (*store_true*): enable layer normalisation +* `--use_huber` (*store_true*): specifies whether to use the huber distance + function as the loss for the critic. If set to False, the mean-squared error + metric is used instead") +* `--actor_update_freq` (*int*): number of training steps per actor policy + update step. The critic policy is updated every training step. Only used when + the algorithm is set to "TD3". Defaults to 2. +* `--noise` (*float*): scaling term to the range of the action space, that is + subsequently used as the standard deviation of Gaussian noise added to the + action if `apply_noise` is set to True in `get_action`". Only used when the + algorithm is set to "TD3". Defaults to 0.1. +* `--target_policy_noise` (*float*): standard deviation term to the noise from + the output of the target actor policy. See TD3 paper for more. Only used when + the algorithm is set to "TD3". Defaults to 0.2. +* `--target_noise_clip` (*float*): clipping term for the noise injected in the + target actor policy. Only used when the algorithm is set to "TD3". Defaults + to 0.5. +* `--target_entropy` (*float*): target entropy used when learning the entropy + coefficient. If set to None, a heuristic value is used. Only used when the + algorithm is set to "SAC". Defaults to None. + +Additionally, the following arguments can be passed when training a multiagent +policy: + +* `--shared` (*store_true*): whether to use a shared policy for all agents +* `--maddpg` (*store_true*): whether to use an algorithm-specific variant of + the MADDPG algorithm ## Simulated Examples -The following networks are available for simulation within flow, and -specifically the **examples/sumo** folder. Similar networks are available with -trainable variants in the examples/rllib and examples/aimsun folders; however, -they may be under different names. +The following networks are available for simulation within flow. These examples are +all available as non-RL examples, while some of them are also available (with +trainable variants) as RL examples, with RLlib or Stable Baselines. + ### bay_bridge.py \& bay_bridge_toll.py @@ -45,7 +203,7 @@ only on the toll booth and sections of the road leading up to it. ![](https://raw.githubusercontent.com/flow-project/flow/master/docs/img/bay_bridge.gif) -### bottlenecks.py +### bottleneck.py Example demonstrating formation of congestion in bottleneck @@ -73,9 +231,15 @@ Example of an open multi-lane network with human-driven vehicles. ![](https://raw.githubusercontent.com/flow-project/flow/master/docs/img/highway.gif) +### highway_ramps.py + +Example of a highway section network with on/off ramps + +![](picture to be added) + ### merge.py -Example of a merge network with human-driven vehicles. +Example of a straight road with an on-ramp merge. In the absence of autonomous vehicles, the network exhibits properties of convective instability, with perturbations propagating upstream from the merge @@ -85,15 +249,16 @@ point before exiting the network. ### minicity.py -Example of modified minicity of University of Delaware network with -human-driven vehicles. +Example of modified mini city developed under a +[collaboration with University of Delaware](https://sites.google.com/view/iccps-policy-transfer), +with human-driven vehicles. ![](https://raw.githubusercontent.com/flow-project/flow/master/docs/img/minicity.gif) -### sugiyama.py +### ring.py -Used as an example of sugiyama experiment. +Used as an example of a ring experiment. -This example consists of 22 IDM cars on a ring road creating shockwaves. +This example consists of 22 IDM cars driving on a ring road creating shockwaves. ![](https://raw.githubusercontent.com/flow-project/flow/master/docs/img/sugiyama.gif) diff --git a/examples/aimsun/bottlenecks.py b/examples/aimsun/bottlenecks.py deleted file mode 100644 index eabd2c135..000000000 --- a/examples/aimsun/bottlenecks.py +++ /dev/null @@ -1,108 +0,0 @@ -"""File demonstrating formation of congestion in bottleneck.""" - -from flow.core.params import AimsunParams, EnvParams, NetParams, \ - InitialConfig, InFlows -from flow.core.params import VehicleParams -from flow.core.params import TrafficLightParams - -from flow.networks.bottleneck import BottleneckNetwork -from flow.envs.bottleneck import BottleneckEnv -from flow.core.experiment import Experiment - -SCALING = 1 -DISABLE_TB = True -# If set to False, ALINEA will control the ramp meter -DISABLE_RAMP_METER = True -INFLOW = 1800 - - -def bottleneck_example(flow_rate, horizon, restart_instance=False, - render=None): - """ - Perform a simulation of vehicles on a bottleneck. - - Parameters - ---------- - flow_rate : float - total inflow rate of vehicles into the bottleneck - horizon : int - time horizon - restart_instance: bool, optional - whether to restart the instance upon reset - render: bool, optional - specifies whether to use the gui during execution - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles on a bottleneck. - """ - if render is None: - render = False - - sim_params = AimsunParams( - sim_step=0.5, - render=render, - restart_instance=restart_instance) - - vehicles = VehicleParams() - - vehicles.add( - veh_id="human", - num_vehicles=1) - - additional_env_params = { - "target_velocity": 40, - "max_accel": 1, - "max_decel": 1, - "lane_change_duration": 5, - "add_rl_if_exit": False, - "disable_tb": DISABLE_TB, - "disable_ramp_metering": DISABLE_RAMP_METER - } - env_params = EnvParams( - horizon=horizon, additional_params=additional_env_params) - - inflow = InFlows() - inflow.add( - veh_type="human", - edge="1", - vehsPerHour=flow_rate, - departLane="random", - departSpeed=10) - - traffic_lights = TrafficLightParams() - if not DISABLE_TB: - traffic_lights.add(node_id="2") - if not DISABLE_RAMP_METER: - traffic_lights.add(node_id="3") - - additional_net_params = {"scaling": SCALING, "speed_limit": 30/3.6} - net_params = NetParams( - inflows=inflow, - additional_params=additional_net_params) - - initial_config = InitialConfig( - spacing="random", - min_gap=5, - lanes_distribution=float("inf"), - edges_distribution=["2", "3", "4", "5"]) - - network = BottleneckNetwork( - name="bay_bridge_toll", - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config, - traffic_lights=traffic_lights) - - env = BottleneckEnv(env_params, sim_params, network, simulator='aimsun') - - return Experiment(env) - - -if __name__ == '__main__': - # import the experiment variable - # inflow, number of steps, binary - exp = bottleneck_example(INFLOW, 1000, render=True) - exp.run(5, 1000) diff --git a/examples/aimsun/figure_eight.py b/examples/aimsun/figure_eight.py deleted file mode 100644 index 195388792..000000000 --- a/examples/aimsun/figure_eight.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Example of a figure 8 network with human-driven vehicles. - -Right-of-way dynamics near the intersection causes vehicles to queue up on -either side of the intersection, leading to a significant reduction in the -average speed of vehicles in the network. -""" -from flow.controllers import IDMController -from flow.core.experiment import Experiment -from flow.core.params import AimsunParams, EnvParams, NetParams -from flow.core.params import VehicleParams -from flow.envs import TestEnv -from flow.networks.figure_eight import FigureEightNetwork, ADDITIONAL_NET_PARAMS - - -def figure_eight_example(render=None): - """Perform a simulation of vehicles on a figure eight. - - Parameters - ---------- - render: bool, optional - specifies whether to use the gui during execution - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles on a figure eight. - """ - sim_params = AimsunParams(sim_step=0.5, render=False, emission_path='data') - - if render is not None: - sim_params.render = render - - vehicles = VehicleParams() - vehicles.add( - veh_id="human", - acceleration_controller=(IDMController, {}), - num_vehicles=14) - - env_params = EnvParams() - - net_params = NetParams( - additional_params=ADDITIONAL_NET_PARAMS.copy()) - - network = FigureEightNetwork( - name="figure8", - vehicles=vehicles, - net_params=net_params) - - env = TestEnv(env_params, sim_params, network, simulator='aimsun') - - return Experiment(env) - - -if __name__ == "__main__": - # import the experiment variable - exp = figure_eight_example(render=True) - - # run for a set number of rollouts / time steps - exp.run(1, 1500) diff --git a/examples/aimsun/grid.py b/examples/aimsun/grid.py deleted file mode 100644 index 042d8828e..000000000 --- a/examples/aimsun/grid.py +++ /dev/null @@ -1,121 +0,0 @@ -"""Traffic Light Grid example.""" -from flow.core.experiment import Experiment -from flow.core.params import AimsunParams, EnvParams, InitialConfig, NetParams -from flow.core.params import VehicleParams -from flow.core.params import TrafficLightParams -from flow.envs.ring.accel import AccelEnv, ADDITIONAL_ENV_PARAMS -from flow.networks import TrafficLightGridNetwork - - -def traffic_light_grid_example(render=None): - """ - Perform a simulation of vehicles on a traffic light grid. - - Parameters - ---------- - render: bool, optional - specifies whether to use the gui during execution - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles and balanced traffic lights on a traffic light grid. - """ - inner_length = 300 - long_length = 500 - short_length = 300 - N_ROWS = 2 - N_COLUMNS = 3 - num_cars_left = 20 - num_cars_right = 20 - num_cars_top = 20 - num_cars_bot = 20 - tot_cars = (num_cars_left + num_cars_right) * N_COLUMNS \ - + (num_cars_top + num_cars_bot) * N_ROWS - - grid_array = { - "short_length": short_length, - "inner_length": inner_length, - "long_length": long_length, - "row_num": N_ROWS, - "col_num": N_COLUMNS, - "cars_left": num_cars_left, - "cars_right": num_cars_right, - "cars_top": num_cars_top, - "cars_bot": num_cars_bot - } - - sim_params = AimsunParams(sim_step=0.5, render=True) - - if render is not None: - sim_params.render = render - - vehicles = VehicleParams() - vehicles.add( - veh_id="human", - num_vehicles=tot_cars) - - env_params = EnvParams(additional_params=ADDITIONAL_ENV_PARAMS) - - tl_logic = TrafficLightParams(baseline=False) - phases = [{ - "duration": "31", - "minDur": "8", - "maxDur": "45", - "yellow": "3", - "state": "GGGrrrGGGrrr" - }, { - "duration": "6", - "minDur": "3", - "maxDur": "6", - "yellow": "3", - "state": "yyyrrryyyrrr" - }, { - "duration": "31", - "minDur": "8", - "maxDur": "45", - "yellow": "3", - "state": "rrrGGGrrrGGG" - }, { - "duration": "6", - "minDur": "3", - "maxDur": "6", - "yellow": "3", - "state": "rrryyyrrryyy" - }] - tl_logic.add("center0", phases=phases, programID=1) - tl_logic.add("center1", phases=phases, programID=1) - tl_logic.add("center2", tls_type="actuated", phases=phases, programID=1) - tl_logic.add("center3", phases=phases, programID=1) - tl_logic.add("center4", phases=phases, programID=1) - tl_logic.add("center5", tls_type="actuated", phases=phases, programID=1) - - additional_net_params = { - "grid_array": grid_array, - "speed_limit": 35, - "horizontal_lanes": 1, - "vertical_lanes": 1 - } - net_params = NetParams(additional_params=additional_net_params) - - initial_config = InitialConfig(spacing='custom') - - network = TrafficLightGridNetwork( - name="grid-intersection", - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config, - traffic_lights=tl_logic) - - env = AccelEnv(env_params, sim_params, network, simulator='aimsun') - - return Experiment(env) - - -if __name__ == "__main__": - # import the experiment variable - exp = traffic_light_grid_example() - - # run for a set number of rollouts / time steps - exp.run(1, 1500) diff --git a/examples/aimsun/merge.py b/examples/aimsun/merge.py deleted file mode 100644 index 813e642b4..000000000 --- a/examples/aimsun/merge.py +++ /dev/null @@ -1,96 +0,0 @@ -"""Example of a merge network with human-driven vehicles. - -In the absence of autonomous vehicles, the network exhibits properties of -convective instability, with perturbations propagating upstream from the merge -point before exiting the network. -""" - -import flow.core.params as params -from flow.core.experiment import Experiment -from flow.networks.merge import MergeNetwork, ADDITIONAL_NET_PARAMS -from flow.controllers import IDMController -from flow.envs.merge import MergePOEnv, ADDITIONAL_ENV_PARAMS - -# inflow rate at the highway -HIGHWAY_RATE = 2000 -# inflow rate at the on-ramp -MERGE_RATE = 500 - - -def merge_example(render=None): - """Perform a simulation of vehicles on a merge. - - Parameters - ---------- - render: bool, optional - specifies whether to use the gui during execution - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles on a merge. - """ - sim_params = params.AimsunParams( - render=True, - emission_path="./data/", - sim_step=0.5, - restart_instance=False) - - if render is not None: - sim_params.render = render - - vehicles = params.VehicleParams() - vehicles.add( - veh_id="human", - acceleration_controller=(IDMController, { - "noise": 0.2 - }), - num_vehicles=5) - - env_params = params.EnvParams( - additional_params=ADDITIONAL_ENV_PARAMS, - sims_per_step=5, - warmup_steps=0) - - inflow = params.InFlows() - inflow.add( - veh_type="human", - edge="inflow_highway", - vehs_per_hour=HIGHWAY_RATE, - departLane="free", - departSpeed=10) - inflow.add( - veh_type="human", - edge="inflow_merge", - vehs_per_hour=MERGE_RATE, - departLane="free", - departSpeed=7.5) - - additional_net_params = ADDITIONAL_NET_PARAMS.copy() - additional_net_params["merge_lanes"] = 1 - additional_net_params["highway_lanes"] = 1 - additional_net_params["pre_merge_length"] = 500 - net_params = params.NetParams( - inflows=inflow, - additional_params=additional_net_params) - - initial_config = params.InitialConfig() - - network = MergeNetwork( - name="merge-baseline", - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config) - - env = MergePOEnv(env_params, sim_params, network, simulator='aimsun') - - return Experiment(env) - - -if __name__ == "__main__": - # import the experiment variable - exp = merge_example() - - # run for a set number of rollouts / time steps - exp.run(1, 3600) diff --git a/examples/aimsun/small_template.py b/examples/aimsun/small_template.py deleted file mode 100644 index d70f6632b..000000000 --- a/examples/aimsun/small_template.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Load an already existing Aimsun template and run the simulation.""" - -from flow.core.experiment import Experiment -from flow.core.params import AimsunParams, EnvParams, NetParams -from flow.core.params import VehicleParams -from flow.envs import TestEnv -from flow.networks import Network -from flow.core.params import InFlows -import flow.config as config -import os - -sim_params = AimsunParams( - sim_step=0.1, - render=True, - emission_path='data', - replication_name="Replication 930", - centroid_config_name="Centroid Configuration 910") - -env_params = EnvParams() -vehicles = VehicleParams() - -template_path = os.path.join(config.PROJECT_PATH, - "flow/utils/aimsun/small_template.ang") - -network = Network( - name="aimsun_small_template", - vehicles=vehicles, - net_params=NetParams( - inflows=InFlows(), - template=template_path - ) -) - -env = TestEnv(env_params, sim_params, network, simulator='aimsun') -exp = Experiment(env) -exp.run(1, 3000) diff --git a/examples/aimsun/sugiyama.py b/examples/aimsun/sugiyama.py deleted file mode 100644 index ac0e7d1a4..000000000 --- a/examples/aimsun/sugiyama.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Used as an example of sugiyama experiment. - -This example consists of 22 IDM cars on a ring creating shockwaves. -""" - -from flow.controllers import IDMController -from flow.core.experiment import Experiment -from flow.core.params import AimsunParams, EnvParams, InitialConfig, NetParams -from flow.core.params import VehicleParams -from flow.envs import TestEnv -from flow.networks.ring import RingNetwork, ADDITIONAL_NET_PARAMS - - -def sugiyama_example(render=None): - """Perform a simulation of vehicles on a ring road. - - Parameters - ---------- - render : bool, optional - specifies whether to use the gui during execution - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles on a ring road. - """ - sim_params = AimsunParams(sim_step=0.5, render=True, emission_path='data') - - if render is not None: - sim_params.render = render - - vehicles = VehicleParams() - vehicles.add( - veh_id="idm", - acceleration_controller=(IDMController, {}), - num_vehicles=22) - - env_params = EnvParams() - - additional_net_params = ADDITIONAL_NET_PARAMS.copy() - net_params = NetParams(additional_params=additional_net_params) - - initial_config = InitialConfig(bunching=20) - - network = RingNetwork( - name="sugiyama", - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config) - - env = TestEnv(env_params, sim_params, network, simulator='aimsun') - - return Experiment(env) - - -if __name__ == "__main__": - # import the experiment variable - exp = sugiyama_example() - - # run for a set number of rollouts / time steps - exp.run(1, 3000) diff --git a/examples/exp_configs/non_rl/aimsun_template.py b/examples/exp_configs/non_rl/aimsun_template.py new file mode 100644 index 000000000..7ab4fd13b --- /dev/null +++ b/examples/exp_configs/non_rl/aimsun_template.py @@ -0,0 +1,56 @@ +"""Load an already existing Aimsun template and run the simulation.""" +from flow.core.params import AimsunParams, EnvParams, NetParams +from flow.core.params import VehicleParams +from flow.core.params import InFlows +import flow.config as config +from flow.envs import TestEnv +from flow.networks import Network +import os + + +# no vehicles in the network +vehicles = VehicleParams() + +# path to the imported Aimsun template +template_path = os.path.join(config.PROJECT_PATH, + "flow/utils/aimsun/small_template.ang") + + +flow_params = dict( + # name of the experiment + exp_tag='aimsun_small_template', + + # name of the flow environment the experiment is running on + env_name=TestEnv, + + # name of the network class the experiment is running on + network=Network, + + # simulator that is used by the experiment + simulator='aimsun', + + # Aimsun-related parameters + sim=AimsunParams( + sim_step=0.1, + render=True, + emission_path='data', + replication_name="Replication 930", + centroid_config_name="Centroid Configuration 910" + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=3000, + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=InFlows(), + template=template_path + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, +) diff --git a/examples/exp_configs/non_rl/bay_bridge.py b/examples/exp_configs/non_rl/bay_bridge.py new file mode 100644 index 000000000..f3e0c465f --- /dev/null +++ b/examples/exp_configs/non_rl/bay_bridge.py @@ -0,0 +1,195 @@ +"""Bay Bridge simulation.""" + +import os +import urllib.request + +from flow.core.params import SumoParams, EnvParams, NetParams, InitialConfig, \ + SumoCarFollowingParams, SumoLaneChangeParams, InFlows +from flow.core.params import VehicleParams +from flow.core.params import TrafficLightParams +from flow.networks.bay_bridge import EDGES_DISTRIBUTION +from flow.controllers import SimCarFollowingController, BayBridgeRouter +from flow.envs import BayBridgeEnv +from flow.networks import BayBridgeNetwork + +USE_TRAFFIC_LIGHTS = False +USE_INFLOWS = False + + +TEMPLATE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "bay_bridge.net.xml") + +# download the template from AWS +if USE_TRAFFIC_LIGHTS: + my_url = "http://s3-us-west-1.amazonaws.com/flow.netfiles/" \ + "bay_bridge_TL_all_green.net.xml" +else: + my_url = "http://s3-us-west-1.amazonaws.com/flow.netfiles/" \ + "bay_bridge_junction_fix.net.xml" +my_file = urllib.request.urlopen(my_url) +data_to_write = my_file.read() + +with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), TEMPLATE), + "wb+") as f: + f.write(data_to_write) + + +vehicles = VehicleParams() +vehicles.add( + veh_id="human", + acceleration_controller=(SimCarFollowingController, {}), + routing_controller=(BayBridgeRouter, {}), + car_following_params=SumoCarFollowingParams( + speedDev=0.2, + speed_mode="all_checks", + ), + lane_change_params=SumoLaneChangeParams( + lc_assertive=20, + lc_pushy=0.8, + lc_speed_gain=4.0, + model="LC2013", + lane_change_mode="no_lc_safe", + # lcKeepRight=0.8 + ), + num_vehicles=1400) + +inflow = InFlows() + +if USE_INFLOWS: + # south + inflow.add( + veh_type="human", + edge="183343422", + vehsPerHour=528, + departLane="0", + departSpeed=20) + inflow.add( + veh_type="human", + edge="183343422", + vehsPerHour=864, + departLane="1", + departSpeed=20) + inflow.add( + veh_type="human", + edge="183343422", + vehsPerHour=600, + departLane="2", + departSpeed=20) + + inflow.add( + veh_type="human", + edge="393649534", + probability=0.1, + departLane="0", + departSpeed=20) # no data for this + + # west + inflow.add( + veh_type="human", + edge="11189946", + vehsPerHour=1752, + departLane="0", + departSpeed=20) + inflow.add( + veh_type="human", + edge="11189946", + vehsPerHour=2136, + departLane="1", + departSpeed=20) + inflow.add( + veh_type="human", + edge="11189946", + vehsPerHour=576, + departLane="2", + departSpeed=20) + + # north + inflow.add( + veh_type="human", + edge="28413687#0", + vehsPerHour=2880, + departLane="0", + departSpeed=20) + inflow.add( + veh_type="human", + edge="28413687#0", + vehsPerHour=2328, + departLane="1", + departSpeed=20) + inflow.add( + veh_type="human", + edge="28413687#0", + vehsPerHour=3060, + departLane="2", + departSpeed=20) + inflow.add( + veh_type="human", + edge="11198593", + probability=0.1, + departLane="0", + departSpeed=20) # no data for this + inflow.add( + veh_type="human", + edge="11197889", + probability=0.1, + departLane="0", + departSpeed=20) # no data for this + + # midway through bridge + inflow.add( + veh_type="human", + edge="35536683", + probability=0.1, + departLane="0", + departSpeed=20) # no data for this + + +flow_params = dict( + # name of the experiment + exp_tag='bay_bridge', + + # name of the flow environment the experiment is running on + env_name=BayBridgeEnv, + + # name of the network class the experiment is running on + network=BayBridgeNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + render=True, + sim_step=0.6, + overtake_right=True, + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=1500, + additional_params={}, + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflow, + template=TEMPLATE, + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig( + spacing="uniform", + min_gap=15, + edges_distribution=EDGES_DISTRIBUTION.copy(), + ), + + # traffic lights to be introduced to specific nodes (see + # flow.core.params.TrafficLightParams) + tls=TrafficLightParams(), +) diff --git a/examples/exp_configs/non_rl/bay_bridge_toll.py b/examples/exp_configs/non_rl/bay_bridge_toll.py new file mode 100644 index 000000000..0941823cb --- /dev/null +++ b/examples/exp_configs/non_rl/bay_bridge_toll.py @@ -0,0 +1,125 @@ +"""Bay Bridge toll example.""" + +import os +import urllib.request + +from flow.core.params import SumoParams, EnvParams, NetParams, InitialConfig, \ + SumoLaneChangeParams, SumoCarFollowingParams, InFlows +from flow.core.params import VehicleParams +from flow.networks.bay_bridge_toll import EDGES_DISTRIBUTION +from flow.controllers import SimCarFollowingController, BayBridgeRouter +from flow.envs import BayBridgeEnv +from flow.networks import BayBridgeTollNetwork + +USE_TRAFFIC_LIGHTS = False + +TEMPLATE = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "bottleneck.net.xml") + + +# download the template from AWS +if USE_TRAFFIC_LIGHTS: + my_url = "http://s3-us-west-1.amazonaws.com/flow.netfiles/" \ + "bay_bridge_TL_all_green.net.xml" +else: + my_url = "http://s3-us-west-1.amazonaws.com/flow.netfiles/" \ + "bay_bridge_junction_fix.net.xml" +my_file = urllib.request.urlopen(my_url) +data_to_write = my_file.read() + +with open( + os.path.join(os.path.dirname(os.path.abspath(__file__)), TEMPLATE), + "wb+") as f: + f.write(data_to_write) + + +vehicles = VehicleParams() +vehicles.add( + veh_id="human", + acceleration_controller=(SimCarFollowingController, {}), + routing_controller=(BayBridgeRouter, {}), + car_following_params=SumoCarFollowingParams( + speedDev=0.2, + speed_mode="all_checks", + ), + lane_change_params=SumoLaneChangeParams( + model="LC2013", + lcCooperative=0.2, + lcSpeedGain=15, + lane_change_mode="no_lc_safe", + ), + num_vehicles=50) + + +inflow = InFlows() +inflow.add( + veh_type="human", + edge="393649534", + probability=0.2, + departLane="random", + departSpeed=10) +inflow.add( + veh_type="human", + edge="4757680", + probability=0.2, + departLane="random", + departSpeed=10) +inflow.add( + veh_type="human", + edge="32661316", + probability=0.2, + departLane="random", + departSpeed=10) +inflow.add( + veh_type="human", + edge="90077193#0", + vehs_per_hour=2000, + departLane="random", + departSpeed=10) + + +flow_params = dict( + # name of the experiment + exp_tag='bay_bridge_toll', + + # name of the flow environment the experiment is running on + env_name=BayBridgeEnv, + + # name of the network class the experiment is running on + network=BayBridgeTollNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + render=True, + sim_step=0.4, + overtake_right=True, + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=1500, + additional_params={}, + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflow, + template=TEMPLATE, + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig( + spacing="uniform", + min_gap=15, + edges_distribution=EDGES_DISTRIBUTION.copy(), + ), +) diff --git a/examples/exp_configs/non_rl/bottleneck.py b/examples/exp_configs/non_rl/bottleneck.py new file mode 100644 index 000000000..f959f6d0f --- /dev/null +++ b/examples/exp_configs/non_rl/bottleneck.py @@ -0,0 +1,108 @@ +"""File demonstrating formation of congestion in bottleneck.""" + +from flow.core.params import SumoParams, EnvParams, NetParams, InitialConfig +from flow.core.params import InFlows, SumoLaneChangeParams, SumoCarFollowingParams +from flow.core.params import VehicleParams +from flow.core.params import TrafficLightParams +from flow.controllers import SimLaneChangeController, ContinuousRouter +from flow.envs import BottleneckEnv +from flow.networks import BottleneckNetwork + +SCALING = 1 +DISABLE_TB = True + +# If set to False, ALINEA will control the ramp meter +DISABLE_RAMP_METER = True +INFLOW = 2300 +HORIZON = 1000 + +vehicles = VehicleParams() +vehicles.add( + veh_id="human", + lane_change_controller=(SimLaneChangeController, {}), + routing_controller=(ContinuousRouter, {}), + car_following_params=SumoCarFollowingParams( + speed_mode=25, + ), + lane_change_params=SumoLaneChangeParams( + lane_change_mode=1621, + ), + num_vehicles=1) + +inflow = InFlows() +inflow.add( + veh_type="human", + edge="1", + vehsPerHour=INFLOW, + departLane="random", + departSpeed=10) + +traffic_lights = TrafficLightParams() +if not DISABLE_TB: + traffic_lights.add(node_id="2") +if not DISABLE_RAMP_METER: + traffic_lights.add(node_id="3") + + +flow_params = dict( + # name of the experiment + exp_tag='bay_bridge_toll', + + # name of the flow environment the experiment is running on + env_name=BottleneckEnv, + + # name of the network class the experiment is running on + network=BottleneckNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + sim_step=0.5, + render=False, + overtake_right=False, + restart_instance=False + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=HORIZON, + additional_params={ + "target_velocity": 40, + "max_accel": 1, + "max_decel": 1, + "lane_change_duration": 5, + "add_rl_if_exit": False, + "disable_tb": DISABLE_TB, + "disable_ramp_metering": DISABLE_RAMP_METER + } + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflow, + additional_params={ + "scaling": SCALING, + "speed_limit": 23 + } + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig( + spacing="random", + min_gap=5, + lanes_distribution=float("inf"), + edges_distribution=["2", "3", "4", "5"] + ), + + # traffic lights to be introduced to specific nodes (see + # flow.core.params.TrafficLightParams) + tls=traffic_lights, +) diff --git a/examples/exp_configs/non_rl/figure_eight.py b/examples/exp_configs/non_rl/figure_eight.py new file mode 100755 index 000000000..13444261a --- /dev/null +++ b/examples/exp_configs/non_rl/figure_eight.py @@ -0,0 +1,62 @@ +"""Example of a figure 8 network with human-driven vehicles. + +Right-of-way dynamics near the intersection causes vehicles to queue up on +either side of the intersection, leading to a significant reduction in the +average speed of vehicles in the network. +""" +from flow.controllers import IDMController, StaticLaneChanger, ContinuousRouter +from flow.core.params import SumoParams, EnvParams, NetParams +from flow.core.params import VehicleParams, SumoCarFollowingParams +from flow.envs.ring.accel import ADDITIONAL_ENV_PARAMS +from flow.networks.figure_eight import ADDITIONAL_NET_PARAMS +from flow.envs import AccelEnv +from flow.networks import FigureEightNetwork + +vehicles = VehicleParams() +vehicles.add( + veh_id="idm", + acceleration_controller=(IDMController, {}), + lane_change_controller=(StaticLaneChanger, {}), + routing_controller=(ContinuousRouter, {}), + car_following_params=SumoCarFollowingParams( + speed_mode="obey_safe_speed", + decel=1.5, + ), + initial_speed=0, + num_vehicles=14) + + +flow_params = dict( + # name of the experiment + exp_tag='figure8', + + # name of the flow environment the experiment is running on + env_name=AccelEnv, + + # name of the network class the experiment is running on + network=FigureEightNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + render=True, + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=1500, + additional_params=ADDITIONAL_ENV_PARAMS.copy(), + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + additional_params=ADDITIONAL_NET_PARAMS.copy(), + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, +) diff --git a/examples/exp_configs/non_rl/highway.py b/examples/exp_configs/non_rl/highway.py new file mode 100644 index 000000000..e7505f2d7 --- /dev/null +++ b/examples/exp_configs/non_rl/highway.py @@ -0,0 +1,87 @@ +"""Example of an open multi-lane network with human-driven vehicles.""" + +from flow.controllers import IDMController +from flow.core.params import SumoParams, EnvParams, NetParams, InitialConfig, SumoLaneChangeParams +from flow.core.params import VehicleParams, InFlows +from flow.envs.ring.lane_change_accel import ADDITIONAL_ENV_PARAMS +from flow.networks.highway import HighwayNetwork, ADDITIONAL_NET_PARAMS +from flow.envs import LaneChangeAccelEnv + +vehicles = VehicleParams() +vehicles.add( + veh_id="human", + acceleration_controller=(IDMController, {}), + lane_change_params=SumoLaneChangeParams( + model="SL2015", + lc_sublane=2.0, + ), + num_vehicles=20) +vehicles.add( + veh_id="human2", + acceleration_controller=(IDMController, {}), + lane_change_params=SumoLaneChangeParams( + model="SL2015", + lc_sublane=2.0, + ), + num_vehicles=20) + +env_params = EnvParams(additional_params=ADDITIONAL_ENV_PARAMS) + +inflow = InFlows() +inflow.add( + veh_type="human", + edge="highway_0", + probability=0.25, + departLane="free", + departSpeed=20) +inflow.add( + veh_type="human2", + edge="highway_0", + probability=0.25, + departLane="free", + departSpeed=20) + + +flow_params = dict( + # name of the experiment + exp_tag='highway', + + # name of the flow environment the experiment is running on + env_name=LaneChangeAccelEnv, + + # name of the network class the experiment is running on + network=HighwayNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + render=True, + lateral_resolution=1.0, + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=1500, + additional_params=ADDITIONAL_ENV_PARAMS.copy(), + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflow, + additional_params=ADDITIONAL_NET_PARAMS.copy(), + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig( + spacing="uniform", + shuffle=True, + ), +) diff --git a/examples/exp_configs/non_rl/highway_ramps.py b/examples/exp_configs/non_rl/highway_ramps.py new file mode 100644 index 000000000..7aeba59cb --- /dev/null +++ b/examples/exp_configs/non_rl/highway_ramps.py @@ -0,0 +1,113 @@ +"""Example of a highway section network with on/off ramps.""" + +from flow.core.params import SumoParams, EnvParams, NetParams, InitialConfig +from flow.core.params import SumoCarFollowingParams, SumoLaneChangeParams +from flow.core.params import InFlows, VehicleParams, TrafficLightParams +from flow.networks.highway_ramps import ADDITIONAL_NET_PARAMS +from flow.envs.ring.accel import AccelEnv, ADDITIONAL_ENV_PARAMS +from flow.networks import HighwayRampsNetwork + +additional_net_params = ADDITIONAL_NET_PARAMS.copy() + +# lengths +additional_net_params["highway_length"] = 1200 +additional_net_params["on_ramps_length"] = 200 +additional_net_params["off_ramps_length"] = 200 + +# number of lanes +additional_net_params["highway_lanes"] = 3 +additional_net_params["on_ramps_lanes"] = 1 +additional_net_params["off_ramps_lanes"] = 1 + +# speed limits +additional_net_params["highway_speed"] = 30 +additional_net_params["on_ramps_speed"] = 20 +additional_net_params["off_ramps_speed"] = 20 + +# ramps +additional_net_params["on_ramps_pos"] = [400] +additional_net_params["off_ramps_pos"] = [800] + +# probability of exiting at the next off-ramp +additional_net_params["next_off_ramp_proba"] = 0.25 + +# inflow rates in vehs/hour +HIGHWAY_INFLOW_RATE = 4000 +ON_RAMPS_INFLOW_RATE = 350 + +vehicles = VehicleParams() +vehicles.add( + veh_id="human", + car_following_params=SumoCarFollowingParams( + speed_mode="obey_safe_speed", # for safer behavior at the merges + tau=1.5 # larger distance between cars + ), + lane_change_params=SumoLaneChangeParams(lane_change_mode=1621) +) + +inflows = InFlows() +inflows.add( + veh_type="human", + edge="highway_0", + vehs_per_hour=HIGHWAY_INFLOW_RATE, + depart_lane="free", + depart_speed="max", + name="highway_flow") +for i in range(len(additional_net_params["on_ramps_pos"])): + inflows.add( + veh_type="human", + edge="on_ramp_{}".format(i), + vehs_per_hour=ON_RAMPS_INFLOW_RATE, + depart_lane="first", + depart_speed="max", + name="on_ramp_flow") + + +flow_params = dict( + # name of the experiment + exp_tag='highway-ramp', + + # name of the flow environment the experiment is running on + env_name=AccelEnv, + + # name of the network class the experiment is running on + network=HighwayRampsNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + render=True, + emission_path="./data/", + sim_step=0.2, + restart_instance=True + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + additional_params=ADDITIONAL_ENV_PARAMS, + horizon=3600, + sims_per_step=5, + warmup_steps=0 + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflows, + additional_params=additional_net_params + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig(), + + # traffic lights to be introduced to specific nodes (see + # flow.core.params.TrafficLightParams) + tls=TrafficLightParams(), +) diff --git a/examples/exp_configs/non_rl/highway_single.py b/examples/exp_configs/non_rl/highway_single.py new file mode 100644 index 000000000..0ced89f27 --- /dev/null +++ b/examples/exp_configs/non_rl/highway_single.py @@ -0,0 +1,113 @@ +"""Example of an open network with human-driven vehicles.""" +from flow.controllers import IDMController +from flow.core.params import EnvParams +from flow.core.params import NetParams +from flow.core.params import InitialConfig +from flow.core.params import InFlows +from flow.core.params import VehicleParams +from flow.core.params import SumoParams +from flow.core.params import SumoLaneChangeParams +from flow.core.params import SumoCarFollowingParams +from flow.networks import HighwayNetwork +from flow.envs import TestEnv +from flow.networks.highway import ADDITIONAL_NET_PARAMS + +# the speed of vehicles entering the network +TRAFFIC_SPEED = 24.1 +# the maximum speed at the downstream boundary edge +END_SPEED = 6.0 +# the inflow rate of vehicles +TRAFFIC_FLOW = 2215 +# the simulation time horizon (in steps) +HORIZON = 1500 +# whether to include noise in the car-following models +INCLUDE_NOISE = True + +additional_net_params = ADDITIONAL_NET_PARAMS.copy() +additional_net_params.update({ + # length of the highway + "length": 2500, + # number of lanes + "lanes": 1, + # speed limit for all edges + "speed_limit": 30, + # number of edges to divide the highway into + "num_edges": 2, + # whether to include a ghost edge. This edge is provided a different speed + # limit. + "use_ghost_edge": True, + # speed limit for the ghost edge + "ghost_speed_limit": END_SPEED, + # length of the downstream ghost edge with the reduced speed limit + "boundary_cell_length": 300, +}) + +vehicles = VehicleParams() +vehicles.add( + "human", + acceleration_controller=(IDMController, { + 'a': 1.3, + 'b': 2.0, + 'noise': 0.3 if INCLUDE_NOISE else 0.0 + }), + car_following_params=SumoCarFollowingParams( + min_gap=0.5 + ), + lane_change_params=SumoLaneChangeParams( + model="SL2015", + lc_sublane=2.0, + ), +) + +inflows = InFlows() +inflows.add( + veh_type="human", + edge="highway_0", + vehs_per_hour=TRAFFIC_FLOW, + depart_lane="free", + depart_speed=TRAFFIC_SPEED, + name="idm_highway_inflow") + +flow_params = dict( + # name of the experiment + exp_tag='highway-single', + + # name of the flow environment the experiment is running on + env_name=TestEnv, + + # name of the network class the experiment is running on + network=HighwayNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=HORIZON, + warmup_steps=500, + sims_per_step=3, + ), + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + sim_step=0.4, + render=False, + use_ballistic=True, + restart_instance=False + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflows, + additional_params=additional_net_params + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig(), +) diff --git a/examples/exp_configs/non_rl/i210_subnetwork.py b/examples/exp_configs/non_rl/i210_subnetwork.py new file mode 100644 index 000000000..b0c58c894 --- /dev/null +++ b/examples/exp_configs/non_rl/i210_subnetwork.py @@ -0,0 +1,173 @@ +"""I-210 subnetwork example.""" +import os +import numpy as np + +from flow.controllers import IDMController +from flow.controllers import I210Router +from flow.core.params import SumoParams +from flow.core.params import EnvParams +from flow.core.params import NetParams +from flow.core.params import SumoLaneChangeParams +from flow.core.params import VehicleParams +from flow.core.params import InitialConfig +from flow.core.params import InFlows +import flow.config as config +from flow.envs import TestEnv +from flow.networks.i210_subnetwork import I210SubNetwork, EDGES_DISTRIBUTION + +# =========================================================================== # +# Specify some configurable constants. # +# =========================================================================== # + +# whether to include the upstream ghost edge in the network +WANT_GHOST_CELL = True +# whether to include the downstream slow-down edge in the network +WANT_DOWNSTREAM_BOUNDARY = True +# whether to include vehicles on the on-ramp +ON_RAMP = False +# the inflow rate of vehicles (in veh/hr) +INFLOW_RATE = 2050 +# the speed of inflowing vehicles from the main edge (in m/s) +INFLOW_SPEED = 25.5 +# horizon over which to run the env +HORIZON = 1500 +# steps to run before follower-stopper is allowed to take control +WARMUP_STEPS = 600 + +# =========================================================================== # +# Specify the path to the network template. # +# =========================================================================== # + +if WANT_DOWNSTREAM_BOUNDARY: + NET_TEMPLATE = os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_" + "downstream.xml") +elif WANT_GHOST_CELL: + NET_TEMPLATE = os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/i210_with_ghost_cell.xml") +else: + NET_TEMPLATE = os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml") + +# If the ghost cell is not being used, remove it from the initial edges that +# vehicles can be placed on. +edges_distribution = EDGES_DISTRIBUTION.copy() +if not WANT_GHOST_CELL: + edges_distribution.remove("ghost0") + +# =========================================================================== # +# Specify vehicle-specific information and inflows. # +# =========================================================================== # + +vehicles = VehicleParams() +vehicles.add( + "human", + num_vehicles=0, + lane_change_params=SumoLaneChangeParams( + lane_change_mode="strategic", + ), + acceleration_controller=(IDMController, { + "a": 1.3, + "b": 2.0, + "noise": 0.3, + }), + routing_controller=(I210Router, {}) if ON_RAMP else None, +) + +inflow = InFlows() +# main highway +for lane in [0, 1, 2, 3, 4]: + inflow.add( + veh_type="human", + edge="ghost0" if WANT_GHOST_CELL else "119257914", + vehs_per_hour=INFLOW_RATE, + departLane=lane, + departSpeed=INFLOW_SPEED) +# on ramp +if ON_RAMP: + inflow.add( + veh_type="human", + edge="27414345", + vehs_per_hour=500, + departLane="random", + departSpeed=10) + inflow.add( + veh_type="human", + edge="27414342#0", + vehs_per_hour=500, + departLane="random", + departSpeed=10) + +# =========================================================================== # +# Generate the flow_params dict with all relevant simulation information. # +# =========================================================================== # + +flow_params = dict( + # name of the experiment + exp_tag='I-210_subnetwork', + + # name of the flow environment the experiment is running on + env_name=TestEnv, + + # name of the network class the experiment is running on + network=I210SubNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # simulation-related parameters + sim=SumoParams( + sim_step=0.4, + render=False, + color_by_speed=True, + use_ballistic=True + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=HORIZON, + warmup_steps=WARMUP_STEPS, + sims_per_step=3 + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflow, + template=NET_TEMPLATE, + additional_params={ + "on_ramp": ON_RAMP, + "ghost_edge": WANT_GHOST_CELL, + } + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig( + edges_distribution=edges_distribution, + ), +) + +# =========================================================================== # +# Specify custom callable that is logged during simulation runtime. # +# =========================================================================== # + +edge_id = "119257908#1-AddedOnRampEdge" +custom_callables = { + "avg_merge_speed": lambda env: np.nan_to_num(np.mean( + env.k.vehicle.get_speed(env.k.vehicle.get_ids_by_edge(edge_id)))), + "avg_outflow": lambda env: np.nan_to_num( + env.k.vehicle.get_outflow_rate(120)), + # we multiply by 5 to account for the vehicle length and by 1000 to convert + # into veh/km + "avg_density": lambda env: 5 * 1000 * len(env.k.vehicle.get_ids_by_edge( + edge_id)) / (env.k.network.edge_length(edge_id) + * env.k.network.num_lanes(edge_id)), +} diff --git a/examples/exp_configs/non_rl/merge.py b/examples/exp_configs/non_rl/merge.py new file mode 100644 index 000000000..2a1ff0455 --- /dev/null +++ b/examples/exp_configs/non_rl/merge.py @@ -0,0 +1,97 @@ +"""Example of a merge network with human-driven vehicles. + +In the absence of autonomous vehicles, the network exhibits properties of +convective instability, with perturbations propagating upstream from the merge +point before exiting the network. +""" + +from flow.core.params import SumoParams, EnvParams, \ + NetParams, InitialConfig, InFlows, SumoCarFollowingParams +from flow.core.params import VehicleParams +from flow.controllers import IDMController +from flow.envs.merge import MergePOEnv, ADDITIONAL_ENV_PARAMS +from flow.networks import MergeNetwork + +# inflow rate at the highway +FLOW_RATE = 2000 + +vehicles = VehicleParams() +vehicles.add( + veh_id="human", + acceleration_controller=(IDMController, { + "noise": 0.2 + }), + car_following_params=SumoCarFollowingParams( + speed_mode="obey_safe_speed", + ), + num_vehicles=5) + +inflow = InFlows() +inflow.add( + veh_type="human", + edge="inflow_highway", + vehs_per_hour=FLOW_RATE, + departLane="free", + departSpeed=10) +inflow.add( + veh_type="human", + edge="inflow_merge", + vehs_per_hour=100, + departLane="free", + departSpeed=7.5) + + +flow_params = dict( + # name of the experiment + exp_tag='merge-baseline', + + # name of the flow environment the experiment is running on + env_name=MergePOEnv, + + # name of the network class the experiment is running on + network=MergeNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + render=True, + emission_path="./data/", + sim_step=0.2, + restart_instance=False, + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=3600, + additional_params=ADDITIONAL_ENV_PARAMS, + sims_per_step=5, + warmup_steps=0, + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflow, + additional_params={ + "merge_length": 100, + "pre_merge_length": 500, + "post_merge_length": 100, + "merge_lanes": 1, + "highway_lanes": 1, + "speed_limit": 30, + }, + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig( + spacing="uniform", + perturbation=5.0, + ), +) diff --git a/examples/exp_configs/non_rl/minicity.py b/examples/exp_configs/non_rl/minicity.py new file mode 100644 index 000000000..35d5edbce --- /dev/null +++ b/examples/exp_configs/non_rl/minicity.py @@ -0,0 +1,79 @@ +"""Example of modified minicity network with human-driven vehicles.""" +from flow.controllers import IDMController +from flow.controllers import RLController +from flow.core.params import SumoParams, EnvParams, NetParams, InitialConfig +from flow.core.params import SumoCarFollowingParams, SumoLaneChangeParams +from flow.core.params import VehicleParams +from flow.envs.ring.accel import AccelEnv, ADDITIONAL_ENV_PARAMS +from flow.controllers.routing_controllers import MinicityRouter +from flow.networks import MiniCityNetwork + + +vehicles = VehicleParams() +vehicles.add( + veh_id="idm", + acceleration_controller=(IDMController, {}), + routing_controller=(MinicityRouter, {}), + car_following_params=SumoCarFollowingParams( + speed_mode=1, + ), + lane_change_params=SumoLaneChangeParams( + lane_change_mode="no_lc_safe", + ), + initial_speed=0, + num_vehicles=90) +vehicles.add( + veh_id="rl", + acceleration_controller=(RLController, {}), + routing_controller=(MinicityRouter, {}), + car_following_params=SumoCarFollowingParams( + speed_mode="obey_safe_speed", + ), + initial_speed=0, + num_vehicles=10) + + +flow_params = dict( + # name of the experiment + exp_tag='minicity', + + # name of the flow environment the experiment is running on + env_name=AccelEnv, + + # name of the network class the experiment is running on + network=MiniCityNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + sim_step=0.25, + render='drgb', + save_render=False, + sight_radius=30, + pxpm=3, + show_radius=True, + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=750, + additional_params=ADDITIONAL_ENV_PARAMS + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams(), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig( + spacing="random", + min_gap=5, + ), +) diff --git a/examples/exp_configs/non_rl/ring.py b/examples/exp_configs/non_rl/ring.py new file mode 100755 index 000000000..bffbdf3ba --- /dev/null +++ b/examples/exp_configs/non_rl/ring.py @@ -0,0 +1,61 @@ +"""Used as an example of ring experiment. + +This example consists of 22 IDM cars on a ring creating shockwaves. +""" + +from flow.controllers import IDMController, ContinuousRouter +from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams +from flow.core.params import VehicleParams +from flow.envs.ring.accel import AccelEnv, ADDITIONAL_ENV_PARAMS +from flow.networks.ring import RingNetwork, ADDITIONAL_NET_PARAMS + + +vehicles = VehicleParams() +vehicles.add( + veh_id="idm", + acceleration_controller=(IDMController, {}), + routing_controller=(ContinuousRouter, {}), + num_vehicles=22) + + +flow_params = dict( + # name of the experiment + exp_tag='ring', + + # name of the flow environment the experiment is running on + env_name=AccelEnv, + + # name of the network class the experiment is running on + network=RingNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + render=True, + sim_step=0.1, + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=1500, + additional_params=ADDITIONAL_ENV_PARAMS, + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + additional_params=ADDITIONAL_NET_PARAMS.copy(), + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig( + bunching=20, + ), +) diff --git a/examples/exp_configs/non_rl/traffic_light_grid.py b/examples/exp_configs/non_rl/traffic_light_grid.py new file mode 100644 index 000000000..2d9894e9d --- /dev/null +++ b/examples/exp_configs/non_rl/traffic_light_grid.py @@ -0,0 +1,235 @@ +"""Grid example.""" +from flow.controllers import GridRouter +from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams +from flow.core.params import VehicleParams +from flow.core.params import TrafficLightParams +from flow.core.params import SumoCarFollowingParams +from flow.core.params import InFlows +from flow.envs.ring.accel import AccelEnv, ADDITIONAL_ENV_PARAMS +from flow.networks import TrafficLightGridNetwork + +USE_INFLOWS = False + +v_enter = 10 +inner_length = 300 +long_length = 500 +short_length = 300 +n_rows = 2 +n_columns = 3 +num_cars_left = 20 +num_cars_right = 20 +num_cars_top = 20 +num_cars_bot = 20 +tot_cars = (num_cars_left + num_cars_right) * n_columns \ + + (num_cars_top + num_cars_bot) * n_rows + +grid_array = { + "short_length": short_length, + "inner_length": inner_length, + "long_length": long_length, + "row_num": n_rows, + "col_num": n_columns, + "cars_left": num_cars_left, + "cars_right": num_cars_right, + "cars_top": num_cars_top, + "cars_bot": num_cars_bot +} + + +def gen_edges(col_num, row_num): + """Generate the names of the outer edges in the grid network. + + Parameters + ---------- + col_num : int + number of columns in the grid + row_num : int + number of rows in the grid + + Returns + ------- + list of str + names of all the outer edges + """ + edges = [] + + # build the left and then the right edges + for i in range(col_num): + edges += ['left' + str(row_num) + '_' + str(i)] + edges += ['right' + '0' + '_' + str(i)] + + # build the bottom and then top edges + for i in range(row_num): + edges += ['bot' + str(i) + '_' + '0'] + edges += ['top' + str(i) + '_' + str(col_num)] + + return edges + + +def get_flow_params(col_num, row_num, additional_net_params): + """Define the network and initial params in the presence of inflows. + + Parameters + ---------- + col_num : int + number of columns in the grid + row_num : int + number of rows in the grid + additional_net_params : dict + network-specific parameters that are unique to the grid + + Returns + ------- + flow.core.params.InitialConfig + parameters specifying the initial configuration of vehicles in the + network + flow.core.params.NetParams + network-specific parameters used to generate the network + """ + initial = InitialConfig( + spacing='custom', lanes_distribution=float('inf'), shuffle=True) + + inflow = InFlows() + outer_edges = gen_edges(col_num, row_num) + for i in range(len(outer_edges)): + inflow.add( + veh_type='human', + edge=outer_edges[i], + probability=0.25, + departLane='free', + departSpeed=20) + + net = NetParams( + inflows=inflow, + additional_params=additional_net_params) + + return initial, net + + +def get_non_flow_params(enter_speed, add_net_params): + """Define the network and initial params in the absence of inflows. + + Note that when a vehicle leaves a network in this case, it is immediately + returns to the start of the row/column it was traversing, and in the same + direction as it was before. + + Parameters + ---------- + enter_speed : float + initial speed of vehicles as they enter the network. + add_net_params: dict + additional network-specific parameters (unique to the grid) + + Returns + ------- + flow.core.params.InitialConfig + parameters specifying the initial configuration of vehicles in the + network + flow.core.params.NetParams + network-specific parameters used to generate the network + """ + additional_init_params = {'enter_speed': enter_speed} + initial = InitialConfig( + spacing='custom', additional_params=additional_init_params) + net = NetParams(additional_params=add_net_params) + + return initial, net + + +vehicles = VehicleParams() +vehicles.add( + veh_id="human", + routing_controller=(GridRouter, {}), + car_following_params=SumoCarFollowingParams( + min_gap=2.5, + decel=7.5, # avoid collisions at emergency stops + ), + num_vehicles=tot_cars) + +env_params = EnvParams(additional_params=ADDITIONAL_ENV_PARAMS) + +tl_logic = TrafficLightParams(baseline=False) +phases = [{ + "duration": "31", + "minDur": "8", + "maxDur": "45", + "state": "GrGrGrGrGrGr" +}, { + "duration": "6", + "minDur": "3", + "maxDur": "6", + "state": "yryryryryryr" +}, { + "duration": "31", + "minDur": "8", + "maxDur": "45", + "state": "rGrGrGrGrGrG" +}, { + "duration": "6", + "minDur": "3", + "maxDur": "6", + "state": "ryryryryryry" +}] +tl_logic.add("center0", phases=phases, programID=1) +tl_logic.add("center1", phases=phases, programID=1) +tl_logic.add("center2", phases=phases, programID=1, tls_type="actuated") + +additional_net_params = { + "grid_array": grid_array, + "speed_limit": 35, + "horizontal_lanes": 1, + "vertical_lanes": 1 +} + +if USE_INFLOWS: + initial_config, net_params = get_flow_params( + col_num=n_columns, + row_num=n_rows, + additional_net_params=additional_net_params) +else: + initial_config, net_params = get_non_flow_params( + enter_speed=v_enter, + add_net_params=additional_net_params) + + +flow_params = dict( + # name of the experiment + exp_tag='grid-intersection', + + # name of the flow environment the experiment is running on + env_name=AccelEnv, + + # name of the network class the experiment is running on + network=TrafficLightGridNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + sim_step=0.1, + render=True, + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=1500, + additional_params=ADDITIONAL_ENV_PARAMS.copy(), + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=net_params, + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=initial_config, + + # traffic lights to be introduced to specific nodes (see + # flow.core.params.TrafficLightParams) + tls=tl_logic, +) diff --git a/examples/rllib/multiagent_exps/multiagent_figure_eight.py b/examples/exp_configs/rl/multiagent/adversarial_figure_eight.py similarity index 51% rename from examples/rllib/multiagent_exps/multiagent_figure_eight.py rename to examples/exp_configs/rl/multiagent/adversarial_figure_eight.py index c07739d26..4fb81ce97 100644 --- a/examples/rllib/multiagent_exps/multiagent_figure_eight.py +++ b/examples/exp_configs/rl/multiagent/adversarial_figure_eight.py @@ -8,20 +8,7 @@ # the negative of the AV reward from copy import deepcopy -import json - -import ray -try: - from ray.rllib.agents.agent import get_agent_class -except ImportError: - from ray.rllib.agents.registry import get_agent_class from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy -from ray import tune -from ray.tune.registry import register_env -from ray.tune import run_experiments - -from flow.envs.multiagent import MultiAgentAccelEnv -from flow.networks import FigureEightNetwork from flow.controllers import ContinuousRouter from flow.controllers import IDMController from flow.controllers import RLController @@ -32,8 +19,10 @@ from flow.core.params import SumoCarFollowingParams from flow.core.params import VehicleParams from flow.networks.figure_eight import ADDITIONAL_NET_PARAMS +from flow.envs.multiagent import AdversarialAccelEnv +from flow.networks import FigureEightNetwork from flow.utils.registry import make_create_env -from flow.utils.rllib import FlowParamsEncoder +from ray.tune.registry import register_env # time horizon of a single rollout HORIZON = 1500 @@ -41,6 +30,10 @@ N_ROLLOUTS = 4 # number of parallel workers N_CPUS = 2 +# number of human-driven vehicles +N_HUMANS = 13 +# number of automated vehicles +N_AVS = 1 # We place one autonomous vehicle and 13 human-driven vehicles in the network vehicles = VehicleParams() @@ -53,7 +46,7 @@ car_following_params=SumoCarFollowingParams( speed_mode='obey_safe_speed', ), - num_vehicles=13) + num_vehicles=N_HUMANS) vehicles.add( veh_id='rl', acceleration_controller=(RLController, {}), @@ -61,14 +54,14 @@ car_following_params=SumoCarFollowingParams( speed_mode='obey_safe_speed', ), - num_vehicles=1) + num_vehicles=N_AVS) flow_params = dict( # name of the experiment - exp_tag='multiagent_figure_eight', + exp_tag='adversarial_figure_eight', # name of the flow environment the experiment is running on - env_name=MultiAgentAccelEnv, + env_name=AdversarialAccelEnv, # name of the network class the experiment is running on network=FigureEightNetwork, @@ -110,82 +103,25 @@ ) -def setup_exps(): - """Return the relevant components of an RLlib experiment. - - Returns - ------- - str - name of the training algorithm - str - name of the gym environment to be trained - dict - training configuration parameters - """ - alg_run = 'PPO' - agent_cls = get_agent_class(alg_run) - config = agent_cls._default_config.copy() - config['num_workers'] = N_CPUS - config['train_batch_size'] = HORIZON * N_ROLLOUTS - config['simple_optimizer'] = True - config['gamma'] = 0.999 # discount rate - config['model'].update({'fcnet_hiddens': [100, 50, 25]}) - config['use_gae'] = True - config['lambda'] = 0.97 - config['sgd_minibatch_size'] = 128 - config['kl_target'] = 0.02 - config['num_sgd_iter'] = 10 - config['horizon'] = HORIZON - config['clip_actions'] = False # FIXME(ev) temporary ray bug - config['observation_filter'] = 'NoFilter' - - # save the flow params for replay - flow_json = json.dumps( - flow_params, cls=FlowParamsEncoder, sort_keys=True, indent=4) - config['env_config']['flow_params'] = flow_json - config['env_config']['run'] = alg_run - - create_env, env_name = make_create_env(params=flow_params, version=0) - - # Register as rllib env - register_env(env_name, create_env) - - test_env = create_env() - obs_space = test_env.observation_space - act_space = test_env.action_space - - def gen_policy(): - return PPOTFPolicy, obs_space, act_space, {} - - # Setup PG with an ensemble of `num_policies` different policy graphs - policy_graphs = {'av': gen_policy(), 'adversary': gen_policy()} - - def policy_mapping_fn(agent_id): - return agent_id - - config.update({ - 'multiagent': { - 'policies': policy_graphs, - 'policy_mapping_fn': tune.function(policy_mapping_fn) - } - }) - return alg_run, env_name, config - - -if __name__ == '__main__': - - alg_run, env_name, config = setup_exps() - ray.init(num_cpus=N_CPUS+1) - - run_experiments({ - flow_params['exp_tag']: { - 'run': alg_run, - 'env': env_name, - 'checkpoint_freq': 1, - 'stop': { - 'training_iteration': 1 - }, - 'config': config, - # 'upload_dir': 's3://' - }, - }) +create_env, env_name = make_create_env(params=flow_params, version=0) + +# Register as rllib env +register_env(env_name, create_env) + +test_env = create_env() +obs_space = test_env.observation_space +act_space = test_env.action_space + + +def gen_policy(): + """Generate a policy in RLlib.""" + return PPOTFPolicy, obs_space, act_space, {} + + +# Setup PG with an ensemble of `num_policies` different policy graphs +POLICY_GRAPHS = {'av': gen_policy(), 'adversary': gen_policy()} + + +def policy_mapping_fn(agent_id): + """Map a policy in RLlib.""" + return agent_id diff --git a/examples/rllib/multiagent_exps/multiagent_stabilizing_the_ring.py b/examples/exp_configs/rl/multiagent/lord_of_the_rings.py similarity index 55% rename from examples/rllib/multiagent_exps/multiagent_stabilizing_the_ring.py rename to examples/exp_configs/rl/multiagent/lord_of_the_rings.py index 300b475c0..e7688c87d 100644 --- a/examples/rllib/multiagent_exps/multiagent_stabilizing_the_ring.py +++ b/examples/exp_configs/rl/multiagent/lord_of_the_rings.py @@ -3,21 +3,7 @@ Creates a set of stabilizing the ring experiments to test if more agents -> fewer needed batches """ - -import json - -import ray -try: - from ray.rllib.agents.agent import get_agent_class -except ImportError: - from ray.rllib.agents.registry import get_agent_class from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy -from ray import tune -from ray.tune.registry import register_env -from ray.tune import run_experiments - -from flow.envs.multiagent import MultiWaveAttenuationPOEnv -from flow.networks import MultiRingNetwork from flow.controllers import ContinuousRouter from flow.controllers import IDMController from flow.controllers import RLController @@ -26,8 +12,10 @@ from flow.core.params import NetParams from flow.core.params import SumoParams from flow.core.params import VehicleParams +from flow.envs.multiagent import MultiWaveAttenuationPOEnv +from flow.networks import MultiRingNetwork from flow.utils.registry import make_create_env -from flow.utils.rllib import FlowParamsEncoder +from ray.tune.registry import register_env # make sure (sample_batch_size * num_workers ~= train_batch_size) # time horizon of a single rollout @@ -107,79 +95,28 @@ ) -def setup_exps(): - """Return the relevant components of an RLlib experiment. - - Returns - ------- - str - name of the training algorithm - str - name of the gym environment to be trained - dict - training configuration parameters - """ - alg_run = 'PPO' - agent_cls = get_agent_class(alg_run) - config = agent_cls._default_config.copy() - config['num_workers'] = N_CPUS - config['train_batch_size'] = HORIZON * N_ROLLOUTS - config['simple_optimizer'] = True - config['gamma'] = 0.999 # discount rate - config['model'].update({'fcnet_hiddens': [32, 32]}) - config['lr'] = tune.grid_search([1e-5]) - config['horizon'] = HORIZON - config['clip_actions'] = False # FIXME(ev) temporary ray bug - config['observation_filter'] = 'NoFilter' - - # save the flow params for replay - flow_json = json.dumps( - flow_params, cls=FlowParamsEncoder, sort_keys=True, indent=4) - config['env_config']['flow_params'] = flow_json - config['env_config']['run'] = alg_run - - create_env, env_name = make_create_env(params=flow_params, version=0) - - # Register as rllib env - register_env(env_name, create_env) - - test_env = create_env() - obs_space = test_env.observation_space - act_space = test_env.action_space - - def gen_policy(): - return PPOTFPolicy, obs_space, act_space, {} - - # Setup PG with an ensemble of `num_policies` different policy graphs - policy_graphs = {'av': gen_policy()} - - def policy_mapping_fn(_): - return 'av' - - config.update({ - 'multiagent': { - 'policies': policy_graphs, - 'policy_mapping_fn': tune.function(policy_mapping_fn), - 'policies_to_train': ['av'] - } - }) - - return alg_run, env_name, config - - -if __name__ == '__main__': - alg_run, env_name, config = setup_exps() - ray.init(num_cpus=N_CPUS + 1) - - run_experiments({ - flow_params['exp_tag']: { - 'run': alg_run, - 'env': env_name, - 'checkpoint_freq': 1, - 'stop': { - 'training_iteration': 1 - }, - 'config': config, - # 'upload_dir': 's3://' - }, - }) +create_env, env_name = make_create_env(params=flow_params, version=0) + +# Register as rllib env +register_env(env_name, create_env) + +test_env = create_env() +obs_space = test_env.observation_space +act_space = test_env.action_space + + +def gen_policy(): + """Generate a policy in RLlib.""" + return PPOTFPolicy, obs_space, act_space, {} + + +# Setup PG with an ensemble of `num_policies` different policy graphs +POLICY_GRAPHS = {'av': gen_policy()} + + +def policy_mapping_fn(_): + """Map a policy in RLlib.""" + return 'av' + + +POLICIES_TO_TRAIN = ['av'] diff --git a/examples/exp_configs/rl/multiagent/multiagent_figure_eight.py b/examples/exp_configs/rl/multiagent/multiagent_figure_eight.py new file mode 100644 index 000000000..0579bb978 --- /dev/null +++ b/examples/exp_configs/rl/multiagent/multiagent_figure_eight.py @@ -0,0 +1,134 @@ +"""Figure eight example.""" +from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.tune.registry import register_env + +from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams +from flow.core.params import VehicleParams, SumoCarFollowingParams +from flow.controllers import IDMController, ContinuousRouter, RLController +from flow.networks.figure_eight import ADDITIONAL_NET_PARAMS +from flow.envs.multiagent import MultiAgentAccelPOEnv +from flow.networks import FigureEightNetwork +from flow.utils.registry import make_create_env + +# time horizon of a single rollout +HORIZON = 1500 +# number of rollouts per training iteration +N_ROLLOUTS = 20 +# number of parallel workers +N_CPUS = 2 + +# desired velocity for all vehicles in the network, in m/s +TARGET_VELOCITY = 20 +# maximum acceleration for autonomous vehicles, in m/s^2 +MAX_ACCEL = 3 +# maximum deceleration for autonomous vehicles, in m/s^2 +MAX_DECEL = 3 + +# number of automated vehicles. Must be one of [1, 2, 7, 14] +NUM_AUTOMATED = 2 + + +assert NUM_AUTOMATED in [1, 2, 7, 14], \ + "num_automated must be one of [1, 2, 7 14]" + + +# We evenly distribute the autonomous vehicles in between the human-driven +# vehicles in the network. +num_human = 14 - NUM_AUTOMATED +human_per_automated = int(num_human / NUM_AUTOMATED) + +vehicles = VehicleParams() +for i in range(NUM_AUTOMATED): + vehicles.add( + veh_id='human_{}'.format(i), + acceleration_controller=(IDMController, { + 'noise': 0.2 + }), + routing_controller=(ContinuousRouter, {}), + car_following_params=SumoCarFollowingParams( + speed_mode="obey_safe_speed", + decel=1.5, + ), + num_vehicles=human_per_automated) + vehicles.add( + veh_id='rl_{}'.format(i), + acceleration_controller=(RLController, {}), + routing_controller=(ContinuousRouter, {}), + car_following_params=SumoCarFollowingParams( + speed_mode="obey_safe_speed", + accel=MAX_ACCEL, + decel=MAX_DECEL, + ), + num_vehicles=1) + +flow_params = dict( + # name of the experiment + exp_tag='multiagent_figure_eight', + + # name of the flow environment the experiment is running on + env_name=MultiAgentAccelPOEnv, + + # name of the network class the experiment is running on + network=FigureEightNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + sim_step=0.1, + render=False, + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=HORIZON, + additional_params={ + 'target_velocity': TARGET_VELOCITY, + 'max_accel': MAX_ACCEL, + 'max_decel': MAX_DECEL, + 'sort_vehicles': False + }, + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + additional_params=ADDITIONAL_NET_PARAMS.copy(), + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig(), +) + + +create_env, env_name = make_create_env(params=flow_params, version=0) + +# Register as rllib env +register_env(env_name, create_env) + +test_env = create_env() +obs_space = test_env.observation_space +act_space = test_env.action_space + + +def gen_policy(): + """Generate a policy in RLlib.""" + return PPOTFPolicy, obs_space, act_space, {} + + +# Setup PG with an ensemble of `num_policies` different policy graphs +POLICY_GRAPHS = {'av': gen_policy()} + + +def policy_mapping_fn(_): + """Map a policy in RLlib.""" + return 'av' + + +POLICIES_TO_TRAIN = ['av'] diff --git a/examples/rllib/multiagent_exps/multiagent_highway.py b/examples/exp_configs/rl/multiagent/multiagent_highway.py similarity index 61% rename from examples/rllib/multiagent_exps/multiagent_highway.py rename to examples/exp_configs/rl/multiagent/multiagent_highway.py index 833614568..cec0b3fba 100644 --- a/examples/rllib/multiagent_exps/multiagent_highway.py +++ b/examples/exp_configs/rl/multiagent/multiagent_highway.py @@ -3,29 +3,17 @@ Trains a non-constant number of agents, all sharing the same policy, on the highway with ramps network. """ -import json -import ray -try: - from ray.rllib.agents.agent import get_agent_class -except ImportError: - from ray.rllib.agents.registry import get_agent_class from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy -from ray import tune -from ray.tune.registry import register_env -from ray.tune import run_experiments - from flow.controllers import RLController from flow.core.params import EnvParams, NetParams, InitialConfig, InFlows, \ VehicleParams, SumoParams, \ SumoCarFollowingParams, SumoLaneChangeParams - -from flow.utils.registry import make_create_env -from flow.utils.rllib import FlowParamsEncoder - -from flow.envs.multiagent import MultiAgentHighwayPOEnv from flow.envs.ring.accel import ADDITIONAL_ENV_PARAMS from flow.networks import HighwayRampsNetwork +from flow.envs.multiagent import MultiAgentHighwayPOEnv from flow.networks.highway_ramps import ADDITIONAL_NET_PARAMS +from flow.utils.registry import make_create_env +from ray.tune.registry import register_env # SET UP PARAMETERS FOR THE SIMULATION @@ -135,11 +123,19 @@ # SET UP FLOW PARAMETERS flow_params = dict( + # name of the experiment exp_tag='multiagent_highway', + + # name of the flow environment the experiment is running on env_name=MultiAgentHighwayPOEnv, + + # name of the network class the experiment is running on network=HighwayRampsNetwork, + + # simulator that is used by the experiment simulator='traci', + # environment related parameters (see flow.core.params.EnvParams) env=EnvParams( horizon=HORIZON, warmup_steps=200, @@ -147,100 +143,48 @@ additional_params=additional_env_params, ), + # sumo-related parameters (see flow.core.params.SumoParams) sim=SumoParams( sim_step=0.2, render=False, restart_instance=True ), + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) net=NetParams( inflows=inflows, - additional_params=additional_net_params), + additional_params=additional_net_params + ), + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) initial=InitialConfig(), ) -# SET UP EXPERIMENT - -def setup_exps(flow_params): - """Create the relevant components of a multiagent RLlib experiment. - - Parameters - ---------- - flow_params : dict - input flow-parameters - - Returns - ------- - str - name of the training algorithm - str - name of the gym environment to be trained - dict - training configuration parameters - """ - alg_run = 'PPO' - agent_cls = get_agent_class(alg_run) - config = agent_cls._default_config.copy() - config['num_workers'] = N_CPUS - config['train_batch_size'] = HORIZON * N_ROLLOUTS - config['simple_optimizer'] = True - config['gamma'] = 0.999 # discount rate - config['model'].update({'fcnet_hiddens': [32, 32]}) - config['lr'] = tune.grid_search([1e-5]) - config['horizon'] = HORIZON - config['clip_actions'] = False - config['observation_filter'] = 'NoFilter' - - # save the flow params for replay - flow_json = json.dumps( - flow_params, cls=FlowParamsEncoder, sort_keys=True, indent=4) - config['env_config']['flow_params'] = flow_json - config['env_config']['run'] = alg_run - - create_env, env_name = make_create_env(params=flow_params, version=0) - - # register as rllib env - register_env(env_name, create_env) - - # multiagent configuration - temp_env = create_env() - policy_graphs = {'av': (PPOTFPolicy, - temp_env.observation_space, - temp_env.action_space, - {})} - - def policy_mapping_fn(_): - return 'av' - - config.update({ - 'multiagent': { - 'policies': policy_graphs, - 'policy_mapping_fn': tune.function(policy_mapping_fn), - 'policies_to_train': ['av'] - } - }) - - return alg_run, env_name, config - - -# RUN EXPERIMENT - -if __name__ == '__main__': - alg_run, env_name, config = setup_exps(flow_params) - ray.init(num_cpus=N_CPUS + 1) - - run_experiments({ - flow_params['exp_tag']: { - 'run': alg_run, - 'env': env_name, - 'checkpoint_freq': 20, - 'checkpoint_at_end': True, - 'stop': { - 'training_iteration': N_TRAINING_ITERATIONS - }, - 'config': config, - }, - }) +# SET UP RLLIB MULTI-AGENT FEATURES + +create_env, env_name = make_create_env(params=flow_params, version=0) + +# register as rllib env +register_env(env_name, create_env) + +# multiagent configuration +test_env = create_env() +obs_space = test_env.observation_space +act_space = test_env.action_space + + +POLICY_GRAPHS = {'av': (PPOTFPolicy, obs_space, act_space, {})} + +POLICIES_TO_TRAIN = ['av'] + + +def policy_mapping_fn(_): + """Map a policy in RLlib.""" + return 'av' diff --git a/examples/exp_configs/rl/multiagent/multiagent_i210.py b/examples/exp_configs/rl/multiagent/multiagent_i210.py new file mode 100644 index 000000000..a6d194708 --- /dev/null +++ b/examples/exp_configs/rl/multiagent/multiagent_i210.py @@ -0,0 +1,189 @@ +"""Multi-agent I-210 example. + +Trains a non-constant number of agents, all sharing the same policy, on the +highway with ramps network. +""" +import os + +from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.tune.registry import register_env + +import flow.config as config +from flow.controllers.rlcontroller import RLController +from flow.core.params import EnvParams +from flow.core.params import NetParams +from flow.core.params import InitialConfig +from flow.core.params import InFlows +from flow.core.params import VehicleParams +from flow.core.params import SumoParams +from flow.core.params import SumoLaneChangeParams +from flow.networks.i210_subnetwork import I210SubNetwork, EDGES_DISTRIBUTION +from flow.envs.multiagent.i210 import I210MultiEnv, ADDITIONAL_ENV_PARAMS +from flow.utils.registry import make_create_env + +# SET UP PARAMETERS FOR THE SIMULATION + +# number of training iterations +N_TRAINING_ITERATIONS = 200 +# number of rollouts per training iteration +N_ROLLOUTS = 2 +# number of steps per rollout +HORIZON = 500 +# number of parallel workers +N_CPUS = 1 + +# percentage of autonomous vehicles compared to human vehicles on highway +PENETRATION_RATE = 10 + +# TODO: temporary fix +edges_distribution = EDGES_DISTRIBUTION.copy() +edges_distribution.remove("ghost0") + +# SET UP PARAMETERS FOR THE ENVIRONMENT +additional_env_params = ADDITIONAL_ENV_PARAMS.copy() +additional_env_params.update({ + 'max_accel': 1, + 'max_decel': 1, + # configure the observation space. Look at the I210MultiEnv class for more info. + 'lead_obs': True, +}) + +# CREATE VEHICLE TYPES AND INFLOWS +# no vehicles in the network +vehicles = VehicleParams() +vehicles.add( + "human", + num_vehicles=0, + lane_change_params=SumoLaneChangeParams( + lane_change_mode="strategic", + ) +) +vehicles.add( + "av", + acceleration_controller=(RLController, {}), + num_vehicles=0, +) + +inflow = InFlows() +# main highway +pen_rate = PENETRATION_RATE / 100 +assert pen_rate < 1.0, "your penetration rate is over 100%" +assert pen_rate > 0.0, "your penetration rate should be above zero" +inflow.add( + veh_type="human", + edge="119257914", + vehs_per_hour=8378 * pen_rate, + # probability=1.0, + departLane="random", + departSpeed=20) +# on ramp +# inflow.add( +# veh_type="human", +# edge="27414345", +# vehs_per_hour=321 * pen_rate, +# departLane="random", +# departSpeed=20) +# inflow.add( +# veh_type="human", +# edge="27414342#0", +# vehs_per_hour=421 * pen_rate, +# departLane="random", +# departSpeed=20) + +# Now add the AVs +# main highway +inflow.add( + veh_type="av", + edge="119257914", + vehs_per_hour=int(8378 * pen_rate), + # probability=1.0, + departLane="random", + departSpeed=20) +# # on ramp +# inflow.add( +# veh_type="av", +# edge="27414345", +# vehs_per_hour=int(321 * pen_rate), +# departLane="random", +# departSpeed=20) +# inflow.add( +# veh_type="av", +# edge="27414342#0", +# vehs_per_hour=int(421 * pen_rate), +# departLane="random", +# departSpeed=20) + +NET_TEMPLATE = os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml") + +flow_params = dict( + # name of the experiment + exp_tag='I_210_subnetwork', + + # name of the flow environment the experiment is running on + env_name=I210MultiEnv, + + # name of the network class the experiment is running on + network=I210SubNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # simulation-related parameters + sim=SumoParams( + sim_step=0.8, + render=False, + color_by_speed=True, + restart_instance=True + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=HORIZON, + sims_per_step=1, + additional_params=additional_env_params, + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflow, + template=NET_TEMPLATE, + additional_params={ + "on_ramp": False, + "ghost_edge": False + } + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig( + edges_distribution=edges_distribution, + ), +) + +# SET UP RLLIB MULTI-AGENT FEATURES + +create_env, env_name = make_create_env(params=flow_params, version=0) + +# register as rllib env +register_env(env_name, create_env) + +# multiagent configuration +test_env = create_env() +obs_space = test_env.observation_space +act_space = test_env.action_space + +POLICY_GRAPHS = {'av': (PPOTFPolicy, obs_space, act_space, {})} + +POLICIES_TO_TRAIN = ['av'] + + +def policy_mapping_fn(_): + """Map a policy in RLlib.""" + return 'av' diff --git a/examples/stable_baselines/stabilizing_highway.py b/examples/exp_configs/rl/multiagent/multiagent_merge.py similarity index 52% rename from examples/stable_baselines/stabilizing_highway.py rename to examples/exp_configs/rl/multiagent/multiagent_merge.py index 0cdb3459f..bfc9fb3b7 100644 --- a/examples/stable_baselines/stabilizing_highway.py +++ b/examples/exp_configs/rl/multiagent/multiagent_merge.py @@ -1,23 +1,19 @@ """Open merge example. Trains a a small percentage of rl vehicles to dissipate shockwaves caused by -merges in an open network. +on-ramp merge to a single lane open highway network. """ -import argparse -import json -import os +from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.tune.registry import register_env -from stable_baselines.common.vec_env import DummyVecEnv, SubprocVecEnv -from stable_baselines import PPO2 - -from flow.envs import MergePOEnv -from flow.networks import MergeNetwork -from flow.core.params import SumoParams, EnvParams, InitialConfig, InFlows, NetParams -from flow.core.params import VehicleParams, SumoCarFollowingParams -from flow.controllers import RLController, IDMController +from flow.core.params import SumoParams, EnvParams, InitialConfig +from flow.core.params import NetParams, InFlows, SumoCarFollowingParams from flow.networks.merge import ADDITIONAL_NET_PARAMS -from flow.utils.registry import env_constructor -from flow.utils.rllib import FlowParamsEncoder, get_flow_params +from flow.core.params import VehicleParams +from flow.controllers import IDMController, RLController +from flow.envs.multiagent import MultiAgentMergePOEnv +from flow.networks import MergeNetwork +from flow.utils.registry import make_create_env # experiment number # - 0: 10% RL penetration, 5 max controllable vehicles @@ -89,10 +85,10 @@ flow_params = dict( # name of the experiment - exp_tag="stabilizing_open_network_merges", + exp_tag="multiagent_merge", # name of the flow environment the experiment is running on - env_name=MergePOEnv, + env_name=MultiAgentMergePOEnv, # name of the network class the experiment is running on network=MergeNetwork, @@ -116,7 +112,6 @@ "max_accel": 1.5, "max_decel": 1.5, "target_velocity": 20, - "num_rl": NUM_RL, }, ), @@ -137,51 +132,28 @@ ) -def run_model(num_cpus=1, rollout_size=50, num_steps=50): - """Run the model for num_steps if provided. The total rollout length is rollout_size.""" - if num_cpus == 1: - constructor = env_constructor(params=flow_params, version=0)() - env = DummyVecEnv([lambda: constructor]) # The algorithms require a vectorized environment to run - else: - env = SubprocVecEnv([env_constructor(params=flow_params, version=i) for i in range(num_cpus)]) - - model = PPO2('MlpPolicy', env, verbose=1, n_steps=rollout_size) - model.learn(total_timesteps=num_steps) - return model - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--num_cpus', type=int, default=1, help='How many CPUs to use') - parser.add_argument('--num_steps', type=int, default=5000, help='How many total steps to perform learning over') - parser.add_argument('--rollout_size', type=int, default=1000, help='How many steps are in a training batch.') - parser.add_argument('--result_name', type=str, default='stabilize_highway', help='Name of saved model') - args = parser.parse_args() - model = run_model(args.num_cpus, args.rollout_size, args.num_steps) - # Save the model to a desired folder and then delete it to demonstrate loading - if not os.path.exists(os.path.realpath(os.path.expanduser('~/baseline_results'))): - os.makedirs(os.path.realpath(os.path.expanduser('~/baseline_results'))) - path = os.path.realpath(os.path.expanduser('~/baseline_results')) - save_path = os.path.join(path, args.result_name) - print('Saving the trained model!') - model.save(save_path) - # dump the flow params - with open(os.path.join(path, args.result_name) + '.json', 'w') as outfile: - json.dump(flow_params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4) - del model - del flow_params - - # Replay the result by loading the model - print('Loading the trained model and testing it out!') - model = PPO2.load(save_path) - flow_params = get_flow_params(os.path.join(path, args.result_name) + '.json') - flow_params['sim'].render = True - env_constructor = env_constructor(params=flow_params, version=0)() - env = DummyVecEnv([lambda: env_constructor]) # The algorithms require a vectorized environment to run - obs = env.reset() - reward = 0 - for i in range(flow_params['env'].horizon): - action, _states = model.predict(obs) - obs, rewards, dones, info = env.step(action) - reward += rewards - print('the final reward is {}'.format(reward)) +create_env, env_name = make_create_env(params=flow_params, version=0) + +# Register as rllib env +register_env(env_name, create_env) + +test_env = create_env() +obs_space = test_env.observation_space +act_space = test_env.action_space + + +def gen_policy(): + """Generate a policy in RLlib.""" + return PPOTFPolicy, obs_space, act_space, {} + + +# Setup PG with an ensemble of `num_policies` different policy graphs +POLICY_GRAPHS = {'av': gen_policy()} + + +def policy_mapping_fn(_): + """Map a policy in RLlib.""" + return 'av' + + +POLICIES_TO_TRAIN = ['av'] diff --git a/examples/exp_configs/rl/multiagent/multiagent_ring.py b/examples/exp_configs/rl/multiagent/multiagent_ring.py new file mode 100644 index 000000000..a789174f4 --- /dev/null +++ b/examples/exp_configs/rl/multiagent/multiagent_ring.py @@ -0,0 +1,130 @@ +"""Ring road example. + +Trains a number of autonomous vehicles to stabilize the flow of 22 vehicles in +a variable length ring road. +""" +from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from ray.tune.registry import register_env + +from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams +from flow.core.params import VehicleParams, SumoCarFollowingParams +from flow.controllers import RLController, IDMController, ContinuousRouter +from flow.envs.multiagent import MultiAgentWaveAttenuationPOEnv +from flow.networks import RingNetwork +from flow.utils.registry import make_create_env + +# time horizon of a single rollout +HORIZON = 3000 +# number of rollouts per training iteration +N_ROLLOUTS = 20 +# number of parallel workers +N_CPUS = 2 +# number of automated vehicles. Must be less than or equal to 22. +NUM_AUTOMATED = 2 + + +# We evenly distribute the automated vehicles in the network. +num_human = 22 - NUM_AUTOMATED +humans_remaining = num_human + +vehicles = VehicleParams() +for i in range(NUM_AUTOMATED): + # Add one automated vehicle. + vehicles.add( + veh_id="rl_{}".format(i), + acceleration_controller=(RLController, {}), + routing_controller=(ContinuousRouter, {}), + num_vehicles=1) + + # Add a fraction of the remaining human vehicles. + vehicles_to_add = round(humans_remaining / (NUM_AUTOMATED - i)) + humans_remaining -= vehicles_to_add + vehicles.add( + veh_id="human_{}".format(i), + acceleration_controller=(IDMController, { + "noise": 0.2 + }), + car_following_params=SumoCarFollowingParams( + min_gap=0 + ), + routing_controller=(ContinuousRouter, {}), + num_vehicles=vehicles_to_add) + + +flow_params = dict( + # name of the experiment + exp_tag="multiagent_ring", + + # name of the flow environment the experiment is running on + env_name=MultiAgentWaveAttenuationPOEnv, + + # name of the network class the experiment is running on + network=RingNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + sim_step=0.1, + render=False, + restart_instance=False + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=HORIZON, + warmup_steps=750, + clip_actions=False, + additional_params={ + "max_accel": 1, + "max_decel": 1, + "ring_length": [220, 270], + }, + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + additional_params={ + "length": 260, + "lanes": 1, + "speed_limit": 30, + "resolution": 40, + }, ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=InitialConfig(), +) + + +create_env, env_name = make_create_env(params=flow_params, version=0) + +# Register as rllib env +register_env(env_name, create_env) + +test_env = create_env() +obs_space = test_env.observation_space +act_space = test_env.action_space + + +def gen_policy(): + """Generate a policy in RLlib.""" + return PPOTFPolicy, obs_space, act_space, {} + + +# Setup PG with an ensemble of `num_policies` different policy graphs +POLICY_GRAPHS = {'av': gen_policy()} + + +def policy_mapping_fn(_): + """Map a policy in RLlib.""" + return 'av' + + +POLICIES_TO_TRAIN = ['av'] diff --git a/examples/exp_configs/rl/multiagent/multiagent_traffic_light_grid.py b/examples/exp_configs/rl/multiagent/multiagent_traffic_light_grid.py new file mode 100644 index 000000000..b8293f638 --- /dev/null +++ b/examples/exp_configs/rl/multiagent/multiagent_traffic_light_grid.py @@ -0,0 +1,156 @@ +"""Multi-agent traffic light example (single shared policy).""" + +from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy +from flow.envs.multiagent import MultiTrafficLightGridPOEnv +from flow.networks import TrafficLightGridNetwork +from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams +from flow.core.params import InFlows, SumoCarFollowingParams, VehicleParams +from flow.controllers import SimCarFollowingController, GridRouter +from ray.tune.registry import register_env +from flow.utils.registry import make_create_env + +# Experiment parameters +N_ROLLOUTS = 63 # number of rollouts per training iteration +N_CPUS = 63 # number of parallel workers + +# Environment parameters +HORIZON = 400 # time horizon of a single rollout +V_ENTER = 30 # enter speed for departing vehicles +INNER_LENGTH = 300 # length of inner edges in the traffic light grid network +LONG_LENGTH = 100 # length of final edge in route +SHORT_LENGTH = 300 # length of edges that vehicles start on +# number of vehicles originating in the left, right, top, and bottom edges +N_LEFT, N_RIGHT, N_TOP, N_BOTTOM = 1, 1, 1, 1 + +EDGE_INFLOW = 300 # inflow rate of vehicles at every edge +N_ROWS = 3 # number of row of bidirectional lanes +N_COLUMNS = 3 # number of columns of bidirectional lanes + + +# we place a sufficient number of vehicles to ensure they confirm with the +# total number specified above. We also use a "right_of_way" speed mode to +# support traffic light compliance +vehicles = VehicleParams() +num_vehicles = (N_LEFT + N_RIGHT) * N_COLUMNS + (N_BOTTOM + N_TOP) * N_ROWS +vehicles.add( + veh_id="human", + acceleration_controller=(SimCarFollowingController, {}), + car_following_params=SumoCarFollowingParams( + min_gap=2.5, + max_speed=V_ENTER, + decel=7.5, # avoid collisions at emergency stops + speed_mode="right_of_way", + ), + routing_controller=(GridRouter, {}), + num_vehicles=num_vehicles) + +# inflows of vehicles are place on all outer edges (listed here) +outer_edges = [] +outer_edges += ["left{}_{}".format(N_ROWS, i) for i in range(N_COLUMNS)] +outer_edges += ["right0_{}".format(i) for i in range(N_ROWS)] +outer_edges += ["bot{}_0".format(i) for i in range(N_ROWS)] +outer_edges += ["top{}_{}".format(i, N_COLUMNS) for i in range(N_ROWS)] + +# equal inflows for each edge (as dictate by the EDGE_INFLOW constant) +inflow = InFlows() +for edge in outer_edges: + inflow.add( + veh_type="human", + edge=edge, + vehs_per_hour=EDGE_INFLOW, + departLane="free", + departSpeed=V_ENTER) + +flow_params = dict( + # name of the experiment + exp_tag="grid_0_{}x{}_i{}_multiagent".format(N_ROWS, N_COLUMNS, EDGE_INFLOW), + + # name of the flow environment the experiment is running on + env_name=MultiTrafficLightGridPOEnv, + + # name of the network class the experiment is running on + network=TrafficLightGridNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=SumoParams( + restart_instance=True, + sim_step=1, + render=False, + ), + + # environment related parameters (see flow.core.params.EnvParams) + env=EnvParams( + horizon=HORIZON, + additional_params={ + "target_velocity": 50, + "switch_time": 3, + "num_observed": 2, + "discrete": False, + "tl_type": "actuated", + "num_local_edges": 4, + "num_local_lights": 4, + }, + ), + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=NetParams( + inflows=inflow, + additional_params={ + "speed_limit": V_ENTER + 5, # inherited from grid0 benchmark + "grid_array": { + "short_length": SHORT_LENGTH, + "inner_length": INNER_LENGTH, + "long_length": LONG_LENGTH, + "row_num": N_ROWS, + "col_num": N_COLUMNS, + "cars_left": N_LEFT, + "cars_right": N_RIGHT, + "cars_top": N_TOP, + "cars_bot": N_BOTTOM, + }, + "horizontal_lanes": 1, + "vertical_lanes": 1, + }, + ), + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization + # or reset (see flow.core.params.InitialConfig) + initial=InitialConfig( + spacing='custom', + shuffle=True, + ), +) + +create_env, env_name = make_create_env(params=flow_params, version=0) + +# Register as rllib env +register_env(env_name, create_env) + +test_env = create_env() +obs_space = test_env.observation_space +act_space = test_env.action_space + + +def gen_policy(): + """Generate a policy in RLlib.""" + return PPOTFPolicy, obs_space, act_space, {} + + +# Setup PG with a single policy graph for all agents +POLICY_GRAPHS = {'av': gen_policy()} + + +def policy_mapping_fn(_): + """Map a policy in RLlib.""" + return 'av' + + +POLICIES_TO_TRAIN = ['av'] diff --git a/examples/rllib/velocity_bottleneck.py b/examples/exp_configs/rl/singleagent/singleagent_bottleneck.py similarity index 68% rename from examples/rllib/velocity_bottleneck.py rename to examples/exp_configs/rl/singleagent/singleagent_bottleneck.py index 1e6a16eae..25bd3b9f3 100644 --- a/examples/rllib/velocity_bottleneck.py +++ b/examples/exp_configs/rl/singleagent/singleagent_bottleneck.py @@ -3,26 +3,14 @@ Bottleneck in which the actions are specifying a desired velocity in a segment of space """ -import json - -import ray -try: - from ray.rllib.agents.agent import get_agent_class -except ImportError: - from ray.rllib.agents.registry import get_agent_class -from ray.tune import run_experiments -from ray.tune.registry import register_env - -from flow.envs import BottleneckDesiredVelocityEnv -from flow.networks import BottleneckNetwork -from flow.utils.registry import make_create_env -from flow.utils.rllib import FlowParamsEncoder from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams, \ InFlows, SumoCarFollowingParams, SumoLaneChangeParams from flow.core.params import TrafficLightParams from flow.core.params import VehicleParams from flow.controllers import RLController, ContinuousRouter, \ SimLaneChangeController +from flow.envs import BottleneckDesiredVelocityEnv +from flow.networks import BottleneckNetwork # time horizon of a single rollout HORIZON = 1000 @@ -161,63 +149,3 @@ # flow.core.params.TrafficLightParams) tls=traffic_lights, ) - - -def setup_exps(): - """Return the relevant components of an RLlib experiment. - - Returns - ------- - str - name of the training algorithm - str - name of the gym environment to be trained - dict - training configuration parameters - """ - alg_run = "PPO" - - agent_cls = get_agent_class(alg_run) - config = agent_cls._default_config.copy() - config["num_workers"] = N_CPUS - config["train_batch_size"] = HORIZON * N_ROLLOUTS - config["gamma"] = 0.999 # discount rate - config["model"].update({"fcnet_hiddens": [64, 64]}) - config["use_gae"] = True - config["lambda"] = 0.97 - config["kl_target"] = 0.02 - config["num_sgd_iter"] = 10 - config['clip_actions'] = False # FIXME(ev) temporary ray bug - config["horizon"] = HORIZON - - # save the flow params for replay - flow_json = json.dumps( - flow_params, cls=FlowParamsEncoder, sort_keys=True, indent=4) - config['env_config']['flow_params'] = flow_json - config['env_config']['run'] = alg_run - - create_env, gym_name = make_create_env(params=flow_params, version=0) - - # Register as rllib env - register_env(gym_name, create_env) - return alg_run, gym_name, config - - -if __name__ == "__main__": - alg_run, gym_name, config = setup_exps() - ray.init(num_cpus=N_CPUS + 1) - trials = run_experiments({ - flow_params["exp_tag"]: { - "run": alg_run, - "env": gym_name, - "config": { - **config - }, - "checkpoint_freq": 20, - "checkpoint_at_end": True, - "max_failures": 999, - "stop": { - "training_iteration": 200, - }, - } - }) diff --git a/examples/rllib/figure_eight.py b/examples/exp_configs/rl/singleagent/singleagent_figure_eight.py similarity index 51% rename from examples/rllib/figure_eight.py rename to examples/exp_configs/rl/singleagent/singleagent_figure_eight.py index 694b36449..58f0f2e27 100644 --- a/examples/rllib/figure_eight.py +++ b/examples/exp_configs/rl/singleagent/singleagent_figure_eight.py @@ -1,24 +1,10 @@ """Figure eight example.""" - -import json - -import ray -try: - from ray.rllib.agents.agent import get_agent_class -except ImportError: - from ray.rllib.agents.registry import get_agent_class -from ray.tune import run_experiments -from ray.tune.registry import register_env - -from flow.envs import AccelEnv -from flow.networks import FigureEightNetwork -from flow.utils.registry import make_create_env -from flow.utils.rllib import FlowParamsEncoder -from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams, \ - SumoCarFollowingParams -from flow.core.params import VehicleParams +from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams +from flow.core.params import VehicleParams, SumoCarFollowingParams from flow.controllers import IDMController, ContinuousRouter, RLController from flow.networks.figure_eight import ADDITIONAL_NET_PARAMS +from flow.envs import AccelEnv +from flow.networks import FigureEightNetwork # time horizon of a single rollout HORIZON = 1500 @@ -52,7 +38,7 @@ flow_params = dict( # name of the experiment - exp_tag='figure_eight_intersection_control', + exp_tag='singleagent_figure_eight', # name of the flow environment the experiment is running on env_name=AccelEnv, @@ -94,62 +80,3 @@ # reset (see flow.core.params.InitialConfig) initial=InitialConfig(), ) - - -def setup_exps(): - """Return the relevant components of an RLlib experiment. - - Returns - ------- - str - name of the training algorithm - str - name of the gym environment to be trained - dict - training configuration parameters - """ - alg_run = 'PPO' - agent_cls = get_agent_class(alg_run) - config = agent_cls._default_config.copy() - config['num_workers'] = N_CPUS - config['train_batch_size'] = HORIZON * N_ROLLOUTS - config['gamma'] = 0.999 # discount rate - config['model'].update({'fcnet_hiddens': [32, 32]}) - config['use_gae'] = True - config['lambda'] = 0.97 - config['kl_target'] = 0.02 - config['num_sgd_iter'] = 10 - config['clip_actions'] = False # FIXME(ev) temporary ray bug - config['horizon'] = HORIZON - - # save the flow params for replay - flow_json = json.dumps( - flow_params, cls=FlowParamsEncoder, sort_keys=True, indent=4) - config['env_config']['flow_params'] = flow_json - config['env_config']['run'] = alg_run - - create_env, gym_name = make_create_env(params=flow_params, version=0) - - # Register as rllib env - register_env(gym_name, create_env) - return alg_run, gym_name, config - - -if __name__ == '__main__': - alg_run, gym_name, config = setup_exps() - ray.init(num_cpus=N_CPUS + 1) - trials = run_experiments({ - flow_params['exp_tag']: { - 'run': alg_run, - 'env': gym_name, - 'config': { - **config - }, - 'checkpoint_freq': 20, - "checkpoint_at_end": True, - 'max_failures': 999, - 'stop': { - 'training_iteration': 200, - }, - } - }) diff --git a/examples/rllib/stabilizing_highway.py b/examples/exp_configs/rl/singleagent/singleagent_merge.py similarity index 63% rename from examples/rllib/stabilizing_highway.py rename to examples/exp_configs/rl/singleagent/singleagent_merge.py index 1a0e27994..6e6053300 100644 --- a/examples/rllib/stabilizing_highway.py +++ b/examples/exp_configs/rl/singleagent/singleagent_merge.py @@ -1,27 +1,15 @@ """Open merge example. Trains a a small percentage of rl vehicles to dissipate shockwaves caused by -merges in an open network. +on-ramp merge to a single lane open highway network. """ -import json - -import ray -try: - from ray.rllib.agents.agent import get_agent_class -except ImportError: - from ray.rllib.agents.registry import get_agent_class -from ray.tune import run_experiments -from ray.tune.registry import register_env - -from flow.envs import MergePOEnv -from flow.networks import MergeNetwork -from flow.utils.registry import make_create_env -from flow.utils.rllib import FlowParamsEncoder -from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams, \ - InFlows, SumoCarFollowingParams +from flow.core.params import SumoParams, EnvParams, InitialConfig +from flow.core.params import NetParams, InFlows, SumoCarFollowingParams from flow.networks.merge import ADDITIONAL_NET_PARAMS from flow.core.params import VehicleParams from flow.controllers import IDMController, RLController +from flow.envs import MergePOEnv +from flow.networks import MergeNetwork # experiment number # - 0: 10% RL penetration, 5 max controllable vehicles @@ -139,63 +127,3 @@ # reset (see flow.core.params.InitialConfig) initial=InitialConfig(), ) - - -def setup_exps(): - """Return the relevant components of an RLlib experiment. - - Returns - ------- - str - name of the training algorithm - str - name of the gym environment to be trained - dict - training configuration parameters - """ - alg_run = "PPO" - - agent_cls = get_agent_class(alg_run) - config = agent_cls._default_config.copy() - config["num_workers"] = N_CPUS - config["train_batch_size"] = HORIZON * N_ROLLOUTS - config["gamma"] = 0.999 # discount rate - config["model"].update({"fcnet_hiddens": [32, 32, 32]}) - config["use_gae"] = True - config["lambda"] = 0.97 - config["kl_target"] = 0.02 - config["num_sgd_iter"] = 10 - config['clip_actions'] = False # FIXME(ev) temporary ray bug - config["horizon"] = HORIZON - - # save the flow params for replay - flow_json = json.dumps( - flow_params, cls=FlowParamsEncoder, sort_keys=True, indent=4) - config['env_config']['flow_params'] = flow_json - config['env_config']['run'] = alg_run - - create_env, gym_name = make_create_env(params=flow_params, version=0) - - # Register as rllib env - register_env(gym_name, create_env) - return alg_run, gym_name, config - - -if __name__ == "__main__": - alg_run, gym_name, config = setup_exps() - ray.init(num_cpus=N_CPUS + 1) - trials = run_experiments({ - flow_params["exp_tag"]: { - "run": alg_run, - "env": gym_name, - "config": { - **config - }, - "checkpoint_freq": 20, - "checkpoint_at_end": True, - "max_failures": 999, - "stop": { - "training_iteration": 200, - }, - } - }) diff --git a/examples/rllib/stabilizing_the_ring.py b/examples/exp_configs/rl/singleagent/singleagent_ring.py similarity index 55% rename from examples/rllib/stabilizing_the_ring.py rename to examples/exp_configs/rl/singleagent/singleagent_ring.py index 7f3c87408..bfe84fa42 100644 --- a/examples/rllib/stabilizing_the_ring.py +++ b/examples/exp_configs/rl/singleagent/singleagent_ring.py @@ -3,24 +3,11 @@ Trains a single autonomous vehicle to stabilize the flow of 21 human-driven vehicles in a variable length ring road. """ - -import json - -import ray -try: - from ray.rllib.agents.agent import get_agent_class -except ImportError: - from ray.rllib.agents.registry import get_agent_class -from ray.tune import run_experiments -from ray.tune.registry import register_env - -from flow.envs import WaveAttenuationPOEnv -from flow.networks import RingNetwork -from flow.utils.registry import make_create_env -from flow.utils.rllib import FlowParamsEncoder from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams from flow.core.params import VehicleParams, SumoCarFollowingParams from flow.controllers import RLController, IDMController, ContinuousRouter +from flow.envs import WaveAttenuationPOEnv +from flow.networks import RingNetwork # time horizon of a single rollout HORIZON = 3000 @@ -64,6 +51,7 @@ sim=SumoParams( sim_step=0.1, render=False, + restart_instance=False ), # environment related parameters (see flow.core.params.EnvParams) @@ -96,63 +84,3 @@ # reset (see flow.core.params.InitialConfig) initial=InitialConfig(), ) - - -def setup_exps(): - """Return the relevant components of an RLlib experiment. - - Returns - ------- - str - name of the training algorithm - str - name of the gym environment to be trained - dict - training configuration parameters - """ - alg_run = "PPO" - - agent_cls = get_agent_class(alg_run) - config = agent_cls._default_config.copy() - config["num_workers"] = N_CPUS - config["train_batch_size"] = HORIZON * N_ROLLOUTS - config["gamma"] = 0.999 # discount rate - config["model"].update({"fcnet_hiddens": [3, 3]}) - config["use_gae"] = True - config["lambda"] = 0.97 - config["kl_target"] = 0.02 - config["num_sgd_iter"] = 10 - config['clip_actions'] = False # FIXME(ev) temporary ray bug - config["horizon"] = HORIZON - - # save the flow params for replay - flow_json = json.dumps( - flow_params, cls=FlowParamsEncoder, sort_keys=True, indent=4) - config['env_config']['flow_params'] = flow_json - config['env_config']['run'] = alg_run - - create_env, gym_name = make_create_env(params=flow_params, version=0) - - # Register as rllib env - register_env(gym_name, create_env) - return alg_run, gym_name, config - - -if __name__ == "__main__": - alg_run, gym_name, config = setup_exps() - ray.init(num_cpus=N_CPUS + 1) - trials = run_experiments({ - flow_params["exp_tag"]: { - "run": alg_run, - "env": gym_name, - "config": { - **config - }, - "checkpoint_freq": 20, - "checkpoint_at_end": True, - "max_failures": 999, - "stop": { - "training_iteration": 200, - }, - } - }) diff --git a/examples/rllib/traffic_light_grid.py b/examples/exp_configs/rl/singleagent/singleagent_traffic_light_grid.py similarity index 66% rename from examples/rllib/traffic_light_grid.py rename to examples/exp_configs/rl/singleagent/singleagent_traffic_light_grid.py index 766cbefc0..085d26be9 100644 --- a/examples/rllib/traffic_light_grid.py +++ b/examples/exp_configs/rl/singleagent/singleagent_traffic_light_grid.py @@ -1,30 +1,20 @@ """Traffic Light Grid example.""" - -import json - -import ray -try: - from ray.rllib.agents.agent import get_agent_class -except ImportError: - from ray.rllib.agents.registry import get_agent_class -from ray.tune import run_experiments -from ray.tune.registry import register_env - -from flow.envs import TrafficLightGridPOEnv -from flow.networks import TrafficLightGridNetwork -from flow.utils.registry import make_create_env -from flow.utils.rllib import FlowParamsEncoder from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams, \ InFlows, SumoCarFollowingParams from flow.core.params import VehicleParams from flow.controllers import SimCarFollowingController, GridRouter +from flow.envs import TrafficLightGridPOEnv +from flow.networks import TrafficLightGridNetwork # time horizon of a single rollout HORIZON = 200 # number of rollouts per training iteration -N_ROLLOUTS = 20 +N_ROLLOUTS = 30 # number of parallel workers N_CPUS = 2 +# set to True if you would like to run the experiment with inflows of vehicles +# from the edges, and False otherwise +USE_INFLOWS = True def gen_edges(col_num, row_num): @@ -55,7 +45,7 @@ def gen_edges(col_num, row_num): return edges -def get_flow_params(col_num, row_num, additional_net_params): +def get_inflow_params(col_num, row_num, additional_net_params): """Define the network and initial params in the presence of inflows. Parameters @@ -178,6 +168,19 @@ def get_non_flow_params(enter_speed, add_net_params): routing_controller=(GridRouter, {}), num_vehicles=tot_cars) +# collect the initialization and network-specific parameters based on the +# choice to use inflows or not +if USE_INFLOWS: + initial_config, net_params = get_inflow_params( + col_num=N_COLUMNS, + row_num=N_ROWS, + additional_net_params=additional_net_params) +else: + initial_config, net_params = get_non_flow_params( + enter_speed=V_ENTER, + add_net_params=additional_net_params) + + flow_params = dict( # name of the experiment exp_tag='traffic_light_grid', @@ -206,7 +209,7 @@ def get_non_flow_params(enter_speed, add_net_params): # network-related parameters (see flow.core.params.NetParams and the # network's documentation or ADDITIONAL_NET_PARAMS component). This is # filled in by the setup_exps method below. - net=None, + net=net_params, # vehicles to be placed in the network at the start of a rollout (see # flow.core.params.VehicleParams) @@ -215,86 +218,5 @@ def get_non_flow_params(enter_speed, add_net_params): # parameters specifying the positioning of vehicles upon initialization/ # reset (see flow.core.params.InitialConfig). This is filled in by the # setup_exps method below. - initial=None, + initial=initial_config, ) - - -def setup_exps(use_inflows=False): - """Return the relevant components of an RLlib experiment. - - Parameters - ---------- - use_inflows : bool, optional - set to True if you would like to run the experiment with inflows of - vehicles from the edges, and False otherwise - - Returns - ------- - str - name of the training algorithm - str - name of the gym environment to be trained - dict - training configuration parameters - """ - # collect the initialization and network-specific parameters based on the - # choice to use inflows or not - if use_inflows: - initial_config, net_params = get_flow_params( - col_num=N_COLUMNS, - row_num=N_ROWS, - additional_net_params=additional_net_params) - else: - initial_config, net_params = get_non_flow_params( - enter_speed=V_ENTER, - add_net_params=additional_net_params) - - # add the new parameters to flow_params - flow_params['initial'] = initial_config - flow_params['net'] = net_params - - alg_run = 'PPO' - - agent_cls = get_agent_class(alg_run) - config = agent_cls._default_config.copy() - config['num_workers'] = N_CPUS - config['train_batch_size'] = HORIZON * N_ROLLOUTS - config['gamma'] = 0.999 # discount rate - config['model'].update({'fcnet_hiddens': [32, 32]}) - config['use_gae'] = True - config['lambda'] = 0.97 - config['kl_target'] = 0.02 - config['num_sgd_iter'] = 10 - config['clip_actions'] = False # FIXME(ev) temporary ray bug - config['horizon'] = HORIZON - - # save the flow params for replay - flow_json = json.dumps( - flow_params, cls=FlowParamsEncoder, sort_keys=True, indent=4) - config['env_config']['flow_params'] = flow_json - config['env_config']['run'] = alg_run - - create_env, gym_name = make_create_env(params=flow_params, version=0) - - # Register as rllib env - register_env(gym_name, create_env) - return alg_run, gym_name, config - - -if __name__ == '__main__': - alg_run, gym_name, config = setup_exps() - ray.init(num_cpus=N_CPUS + 1) - trials = run_experiments({ - flow_params['exp_tag']: { - 'run': alg_run, - 'env': gym_name, - 'config': { - **config - }, - 'checkpoint_freq': 20, - 'max_failures': 999, - 'stop': { - 'training_iteration': 200, - }, - } - }) diff --git a/examples/exp_configs/templates/sumo/i210_with_ghost_cell.xml b/examples/exp_configs/templates/sumo/i210_with_ghost_cell.xml new file mode 100644 index 000000000..66e5a1131 --- /dev/null +++ b/examples/exp_configs/templates/sumo/i210_with_ghost_cell.xml @@ -0,0 +1,5719 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml b/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml new file mode 100644 index 000000000..ee508b730 --- /dev/null +++ b/examples/exp_configs/templates/sumo/i210_with_ghost_cell_with_downstream.xml @@ -0,0 +1,5719 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/exp_configs/templates/sumo/test2.net.xml b/examples/exp_configs/templates/sumo/test2.net.xml new file mode 100644 index 000000000..00e3edcd5 --- /dev/null +++ b/examples/exp_configs/templates/sumo/test2.net.xml @@ -0,0 +1,5677 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/sumo/density_exp.py b/examples/exp_scripts/bottleneck_density_sweep_capacity_diagram.py similarity index 100% rename from examples/sumo/density_exp.py rename to examples/exp_scripts/bottleneck_density_sweep_capacity_diagram.py diff --git a/examples/exp_scripts/compute_210_calibration.py b/examples/exp_scripts/compute_210_calibration.py new file mode 100644 index 000000000..d586a9019 --- /dev/null +++ b/examples/exp_scripts/compute_210_calibration.py @@ -0,0 +1,33 @@ +"""We load the calibrated data from calibrated_values and compute how accurate it is.""" +import numpy as np +import pandas as pd +import pickle as pkl +import os + +if __name__ == '__main__': + with open(os.path.abspath('../calibrated_values/info_dict.pkl'), 'rb') as file: + data = pkl.load(file) + + calibrated_data = pd.read_csv('../calibrated_values/i210_sub_merge_area_reduced.csv') + valid_section = calibrated_data[calibrated_data['oid'] == 8009307] + speeds = valid_section['speed'].to_numpy() / 3.6 # (km/h to m/s) + density = valid_section['density'] + outflow = valid_section['flow'] + + dict_to_idx = {'oid': 0, 'ent': 1, 'flow': 2, 'ttime': 3, + 'speed': 4, 'density': 5, 'lane_changes': 6, 'total_lane_changes': 7} + + errors = [] + # compute the speed errors for a given set of params + for experiment in data: + merge_speed = experiment['avg_merge_speed'] + # now sum it up in segments noting that the sim step is 0.8 + num_steps = int(120 / 0.8) + + step_sizes = np.arange(0, len(merge_speed), num_steps) + # sum up all the slices + summed_slices = np.add.reduceat(merge_speed, step_sizes) / num_steps + # throw away the last point and the first point before the network is formed + error = np.abs(np.mean(summed_slices[:-1] - speeds[:summed_slices.shape[0] - 1])) + errors.append(error) + print(errors) diff --git a/examples/rllib/__init__.py b/examples/rllib/__init__.py deleted file mode 100644 index d48602a8b..000000000 --- a/examples/rllib/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Empty init file to ensure that the examples documentation builds.""" diff --git a/examples/rllib/multiagent_exps/__init__.py b/examples/rllib/multiagent_exps/__init__.py deleted file mode 100644 index d48602a8b..000000000 --- a/examples/rllib/multiagent_exps/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Empty init file to ensure that the examples documentation builds.""" diff --git a/examples/rllib/multiagent_exps/multiagent_traffic_light_grid.py b/examples/rllib/multiagent_exps/multiagent_traffic_light_grid.py deleted file mode 100644 index 476e0763b..000000000 --- a/examples/rllib/multiagent_exps/multiagent_traffic_light_grid.py +++ /dev/null @@ -1,296 +0,0 @@ -"""Multi-agent traffic light example (single shared policy).""" - -import json -import argparse - -import ray -try: - from ray.rllib.agents.agent import get_agent_class -except ImportError: - from ray.rllib.agents.registry import get_agent_class -from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy -from ray import tune -from ray.tune.registry import register_env -from ray.tune import run_experiments - -from flow.envs.multiagent import MultiTrafficLightGridPOEnv -from flow.networks import TrafficLightGridNetwork -from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams -from flow.core.params import InFlows, SumoCarFollowingParams, VehicleParams -from flow.controllers import SimCarFollowingController, GridRouter - -from flow.utils.registry import make_create_env -from flow.utils.rllib import FlowParamsEncoder - -# Experiment parameters -N_ROLLOUTS = 63 # number of rollouts per training iteration -N_CPUS = 63 # number of parallel workers - -# Environment parameters -HORIZON = 400 # time horizon of a single rollout -V_ENTER = 30 # enter speed for departing vehicles -INNER_LENGTH = 300 # length of inner edges in the traffic light grid network -LONG_LENGTH = 100 # length of final edge in route -SHORT_LENGTH = 300 # length of edges that vehicles start on -# number of vehicles originating in the left, right, top, and bottom edges -N_LEFT, N_RIGHT, N_TOP, N_BOTTOM = 1, 1, 1, 1 - - -def make_flow_params(n_rows, n_columns, edge_inflow): - """ - Generate the flow params for the experiment. - - Parameters - ---------- - n_rows : int - number of rows in the traffic light grid - n_columns : int - number of columns in the traffic light grid - edge_inflow : float - - - Returns - ------- - dict - flow_params object - """ - # we place a sufficient number of vehicles to ensure they confirm with the - # total number specified above. We also use a "right_of_way" speed mode to - # support traffic light compliance - vehicles = VehicleParams() - num_vehicles = (N_LEFT + N_RIGHT) * n_columns + (N_BOTTOM + N_TOP) * n_rows - vehicles.add( - veh_id="human", - acceleration_controller=(SimCarFollowingController, {}), - car_following_params=SumoCarFollowingParams( - min_gap=2.5, - max_speed=V_ENTER, - decel=7.5, # avoid collisions at emergency stops - speed_mode="right_of_way", - ), - routing_controller=(GridRouter, {}), - num_vehicles=num_vehicles) - - # inflows of vehicles are place on all outer edges (listed here) - outer_edges = [] - outer_edges += ["left{}_{}".format(n_rows, i) for i in range(n_columns)] - outer_edges += ["right0_{}".format(i) for i in range(n_rows)] - outer_edges += ["bot{}_0".format(i) for i in range(n_rows)] - outer_edges += ["top{}_{}".format(i, n_columns) for i in range(n_rows)] - - # equal inflows for each edge (as dictate by the EDGE_INFLOW constant) - inflow = InFlows() - for edge in outer_edges: - inflow.add( - veh_type="human", - edge=edge, - vehs_per_hour=edge_inflow, - departLane="free", - departSpeed=V_ENTER) - - flow_params = dict( - # name of the experiment - exp_tag="grid_0_{}x{}_i{}_multiagent".format(n_rows, n_columns, - edge_inflow), - - # name of the flow environment the experiment is running on - env_name=MultiTrafficLightGridPOEnv, - - # name of the network class the experiment is running on - network=TrafficLightGridNetwork, - - # simulator that is used by the experiment - simulator='traci', - - # sumo-related parameters (see flow.core.params.SumoParams) - sim=SumoParams( - restart_instance=True, - sim_step=1, - render=False, - ), - - # environment related parameters (see flow.core.params.EnvParams) - env=EnvParams( - horizon=HORIZON, - additional_params={ - "target_velocity": 50, - "switch_time": 3, - "num_observed": 2, - "discrete": False, - "tl_type": "actuated", - "num_local_edges": 4, - "num_local_lights": 4, - }, - ), - - # network-related parameters (see flow.core.params.NetParams and the - # network's documentation or ADDITIONAL_NET_PARAMS component) - net=NetParams( - inflows=inflow, - additional_params={ - "speed_limit": V_ENTER + 5, # inherited from grid0 benchmark - "grid_array": { - "short_length": SHORT_LENGTH, - "inner_length": INNER_LENGTH, - "long_length": LONG_LENGTH, - "row_num": n_rows, - "col_num": n_columns, - "cars_left": N_LEFT, - "cars_right": N_RIGHT, - "cars_top": N_TOP, - "cars_bot": N_BOTTOM, - }, - "horizontal_lanes": 1, - "vertical_lanes": 1, - }, - ), - - # vehicles to be placed in the network at the start of a rollout (see - # flow.core.params.VehicleParams) - veh=vehicles, - - # parameters specifying the positioning of vehicles upon initialization - # or reset (see flow.core.params.InitialConfig) - initial=InitialConfig( - spacing='custom', - shuffle=True, - ), - ) - return flow_params - - -def setup_exps_PPO(flow_params): - """ - Experiment setup with PPO using RLlib. - - Parameters - ---------- - flow_params : dictionary of flow parameters - - Returns - ------- - str - name of the training algorithm - str - name of the gym environment to be trained - dict - training configuration parameters - """ - alg_run = 'PPO' - agent_cls = get_agent_class(alg_run) - config = agent_cls._default_config.copy() - config["num_workers"] = min(N_CPUS, N_ROLLOUTS) - config['train_batch_size'] = HORIZON * N_ROLLOUTS - config['simple_optimizer'] = True - config['gamma'] = 0.999 # discount rate - config['model'].update({'fcnet_hiddens': [32, 32]}) - config['lr'] = tune.grid_search([1e-5, 1e-4, 1e-3]) - config['horizon'] = HORIZON - config['clip_actions'] = False # FIXME(ev) temporary ray bug - config['observation_filter'] = 'NoFilter' - - # save the flow params for replay - flow_json = json.dumps( - flow_params, cls=FlowParamsEncoder, sort_keys=True, indent=4) - config['env_config']['flow_params'] = flow_json - config['env_config']['run'] = alg_run - - create_env, env_name = make_create_env(params=flow_params, version=0) - - # Register as rllib env - register_env(env_name, create_env) - - test_env = create_env() - obs_space = test_env.observation_space - act_space = test_env.action_space - - def gen_policy(): - return PPOTFPolicy, obs_space, act_space, {} - - # Setup PG with a single policy graph for all agents - policy_graphs = {'av': gen_policy()} - - def policy_mapping_fn(_): - return 'av' - - config.update({ - 'multiagent': { - 'policies': policy_graphs, - 'policy_mapping_fn': tune.function(policy_mapping_fn), - 'policies_to_train': ['av'] - } - }) - - return alg_run, env_name, config - - -if __name__ == '__main__': - EXAMPLE_USAGE = """ - example usage: - python multiagent_traffic_light_grid.py --upload_dir= - """ - - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description="[Flow] Issues multi-agent traffic light grid experiment", - epilog=EXAMPLE_USAGE) - - # required input parameters - parser.add_argument("--upload_dir", type=str, - help="S3 Bucket for uploading results.") - - # optional input parameters - parser.add_argument('--run_mode', type=str, default='local', - help="Experiment run mode (local | cluster)") - parser.add_argument('--algo', type=str, default='PPO', - help="RL method to use (PPO)") - parser.add_argument('--num_rows', type=int, default=3, - help="The number of rows in the traffic light grid network.") - parser.add_argument('--num_cols', type=int, default=3, - help="The number of columns in the traffic light grid network.") - parser.add_argument('--inflow_rate', type=int, default=300, - help="The inflow rate (veh/hr) per edge.") - args = parser.parse_args() - - EDGE_INFLOW = args.inflow_rate # inflow rate of vehicles at every edge - N_ROWS = args.num_rows # number of row of bidirectional lanes - N_COLUMNS = args.num_cols # number of columns of bidirectional lanes - - flow_params = make_flow_params(N_ROWS, N_COLUMNS, EDGE_INFLOW) - - upload_dir = args.upload_dir - RUN_MODE = args.run_mode - ALGO = args.algo - - if ALGO == 'PPO': - alg_run, env_name, config = setup_exps_PPO(flow_params) - else: - raise NotImplementedError - - if RUN_MODE == 'local': - ray.init(num_cpus=N_CPUS + 1) - N_ITER = 1 - elif RUN_MODE == 'cluster': - ray.init(redis_address="localhost:6379") - N_ITER = 2000 - - exp_tag = { - 'run': alg_run, - 'env': env_name, - 'checkpoint_freq': 25, - "max_failures": 10, - 'stop': { - 'training_iteration': N_ITER - }, - 'config': config, - "num_samples": 1, - } - - if upload_dir: - exp_tag["upload_dir"] = "s3://{}".format(upload_dir) - - run_experiments( - { - flow_params["exp_tag"]: exp_tag - }, - ) diff --git a/examples/simulate.py b/examples/simulate.py new file mode 100644 index 000000000..d1dcc5a79 --- /dev/null +++ b/examples/simulate.py @@ -0,0 +1,93 @@ +"""Runner script for non-RL simulations in flow. + +Usage + python simulate.py EXP_CONFIG --no_render +""" +import argparse +import sys +import json +import os +from flow.core.experiment import Experiment + +from flow.core.params import AimsunParams +from flow.utils.rllib import FlowParamsEncoder + + +def parse_args(args): + """Parse training options user can specify in command line. + + Returns + ------- + argparse.Namespace + the output parser object + """ + parser = argparse.ArgumentParser( + description="Parse argument used when running a Flow simulation.", + epilog="python simulate.py EXP_CONFIG --num_runs INT --no_render") + + # required input parameters + parser.add_argument( + 'exp_config', type=str, + help='Name of the experiment configuration file, as located in ' + 'exp_configs/non_rl.') + + # optional input parameters + parser.add_argument( + '--num_runs', type=int, default=1, + help='Number of simulations to run. Defaults to 1.') + parser.add_argument( + '--no_render', + action='store_true', + help='Specifies whether to run the simulation during runtime.') + parser.add_argument( + '--aimsun', + action='store_true', + help='Specifies whether to run the simulation using the simulator ' + 'Aimsun. If not specified, the simulator used is SUMO.') + parser.add_argument( + '--gen_emission', + action='store_true', + help='Specifies whether to generate an emission file from the ' + 'simulation.') + + return parser.parse_known_args(args)[0] + + +if __name__ == "__main__": + flags = parse_args(sys.argv[1:]) + + # Get the flow_params object. + module = __import__("exp_configs.non_rl", fromlist=[flags.exp_config]) + flow_params = getattr(module, flags.exp_config).flow_params + + # Get the custom callables for the runner. + if hasattr(getattr(module, flags.exp_config), "custom_callables"): + callables = getattr(module, flags.exp_config).custom_callables + else: + callables = None + + flow_params['sim'].render = not flags.no_render + flow_params['simulator'] = 'aimsun' if flags.aimsun else 'traci' + + # If Aimsun is being called, replace SumoParams with AimsunParams. + if flags.aimsun: + sim_params = AimsunParams() + sim_params.__dict__.update(flow_params['sim'].__dict__) + flow_params['sim'] = sim_params + + # Specify an emission path if they are meant to be generated. + if flags.gen_emission: + flow_params['sim'].emission_path = "./data" + + # Create the flow_params object + fp_ = flow_params['exp_tag'] + dir_ = flow_params['sim'].emission_path + with open(os.path.join(dir_, "{}.json".format(fp_)), 'w') as outfile: + json.dump(flow_params, outfile, + cls=FlowParamsEncoder, sort_keys=True, indent=4) + + # Create the experiment object. + exp = Experiment(flow_params, callables) + + # Run for the specified number of rollouts. + exp.run(flags.num_runs, convert_to_csv=flags.gen_emission) diff --git a/examples/stable_baselines/figure_eight.py b/examples/stable_baselines/figure_eight.py deleted file mode 100644 index 6e34fdf55..000000000 --- a/examples/stable_baselines/figure_eight.py +++ /dev/null @@ -1,146 +0,0 @@ -"""Ring road example. - -Trains a single autonomous vehicle to stabilize the flow of 21 human-driven -vehicles in a variable length ring road. -""" - -import argparse -import json -import os - -from stable_baselines.common.vec_env import DummyVecEnv, SubprocVecEnv -from stable_baselines import PPO2 - -from flow.envs import AccelEnv -from flow.networks import FigureEightNetwork -from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams -from flow.core.params import SumoCarFollowingParams -from flow.core.params import VehicleParams -from flow.controllers import IDMController, ContinuousRouter, RLController -from flow.networks.figure_eight import ADDITIONAL_NET_PARAMS -from flow.utils.registry import env_constructor -from flow.utils.rllib import FlowParamsEncoder, get_flow_params - -# time horizon of a single rollout -HORIZON = 1500 - - -# We place one autonomous vehicle and 13 human-driven vehicles in the network -vehicles = VehicleParams() -vehicles.add( - veh_id='human', - acceleration_controller=(IDMController, { - 'noise': 0.2 - }), - routing_controller=(ContinuousRouter, {}), - car_following_params=SumoCarFollowingParams( - speed_mode="obey_safe_speed", - decel=1.5, - ), - num_vehicles=13) -vehicles.add( - veh_id='rl', - acceleration_controller=(RLController, {}), - routing_controller=(ContinuousRouter, {}), - car_following_params=SumoCarFollowingParams( - speed_mode="obey_safe_speed", - decel=1.5, - ), - num_vehicles=1) - -flow_params = dict( - # name of the experiment - exp_tag='figure_eight_intersection_control', - - # name of the flow environment the experiment is running on - env_name=AccelEnv, - - # name of the network class the experiment is running on - network=FigureEightNetwork, - - # simulator that is used by the experiment - simulator='traci', - - # sumo-related parameters (see flow.core.params.SumoParams) - sim=SumoParams( - sim_step=0.1, - render=False, - ), - - # environment related parameters (see flow.core.params.EnvParams) - env=EnvParams( - horizon=HORIZON, - additional_params={ - 'target_velocity': 20, - 'max_accel': 3, - 'max_decel': 3, - 'sort_vehicles': False - }, - ), - - # network-related parameters (see flow.core.params.NetParams and the - # network's documentation or ADDITIONAL_NET_PARAMS component) - net=NetParams( - additional_params=ADDITIONAL_NET_PARAMS.copy(), - ), - - # vehicles to be placed in the network at the start of a rollout (see - # flow.core.params.VehicleParams) - veh=vehicles, - - # parameters specifying the positioning of vehicles upon initialization/ - # reset (see flow.core.params.InitialConfig) - initial=InitialConfig(), - -) - - -def run_model(num_cpus=1, rollout_size=50, num_steps=50): - """Run the model for num_steps if provided. The total rollout length is rollout_size.""" - if num_cpus == 1: - constructor = env_constructor(params=flow_params, version=0)() - env = DummyVecEnv([lambda: constructor]) # The algorithms require a vectorized environment to run - else: - env = SubprocVecEnv([env_constructor(params=flow_params, version=i) for i in range(num_cpus)]) - - model = PPO2('MlpPolicy', env, verbose=1, n_steps=rollout_size) - model.learn(total_timesteps=num_steps) - return model - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--num_cpus', type=int, default=1, help='How many CPUs to use') - parser.add_argument('--num_steps', type=int, default=5000, help='How many total steps to perform learning over') - parser.add_argument('--rollout_size', type=int, default=1000, help='How many steps are in a training batch.') - parser.add_argument('--result_name', type=str, default='figure_eight', help='Name of saved model') - args = parser.parse_args() - model = run_model(args.num_cpus, args.rollout_size, args.num_steps) - # Save the model to a desired folder and then delete it to demonstrate loading - if not os.path.exists(os.path.realpath(os.path.expanduser('~/baseline_results'))): - os.makedirs(os.path.realpath(os.path.expanduser('~/baseline_results'))) - path = os.path.realpath(os.path.expanduser('~/baseline_results')) - save_path = os.path.join(path, args.result_name) - - print('Saving the trained model!') - model.save(save_path) - # dump the flow params - with open(os.path.join(path, args.result_name) + '.json', 'w') as outfile: - json.dump(flow_params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4) - del model - del flow_params - - # Replay the result by loading the model - print('Loading the trained model and testing it out!') - model = PPO2.load(save_path) - flow_params = get_flow_params(os.path.join(path, args.result_name) + '.json') - flow_params['sim'].render = True - env_constructor = env_constructor(params=flow_params, version=0)() - env = DummyVecEnv([lambda: env_constructor]) # The algorithms require a vectorized environment to run - obs = env.reset() - reward = 0 - for i in range(flow_params['env'].horizon): - action, _states = model.predict(obs) - obs, rewards, dones, info = env.step(action) - reward += rewards - print('the final reward is {}'.format(reward)) diff --git a/examples/stable_baselines/stabilizing_the_ring.py b/examples/stable_baselines/stabilizing_the_ring.py deleted file mode 100644 index 25f39bc1f..000000000 --- a/examples/stable_baselines/stabilizing_the_ring.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Ring road example. - -Trains a single autonomous vehicle to stabilize the flow of 21 human-driven -vehicles in a variable length ring road. -""" - -import argparse -import json -import os - -from stable_baselines.common.vec_env import DummyVecEnv, SubprocVecEnv -from stable_baselines import PPO2 - -from flow.envs import WaveAttenuationPOEnv -from flow.networks import RingNetwork -from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams -from flow.core.params import VehicleParams, SumoCarFollowingParams -from flow.controllers import RLController, IDMController, ContinuousRouter -from flow.utils.registry import env_constructor -from flow.utils.rllib import FlowParamsEncoder, get_flow_params - -# time horizon of a single rollout -HORIZON = 3000 - -# We place one autonomous vehicle and 22 human-driven vehicles in the network -vehicles = VehicleParams() -vehicles.add( - veh_id="human", - acceleration_controller=(IDMController, { - "noise": 0.2 - }), - car_following_params=SumoCarFollowingParams( - min_gap=0 - ), - routing_controller=(ContinuousRouter, {}), - num_vehicles=21) -vehicles.add( - veh_id="rl", - acceleration_controller=(RLController, {}), - routing_controller=(ContinuousRouter, {}), - num_vehicles=1) - -flow_params = dict( - # name of the experiment - exp_tag="stabilizing_the_ring", - - # name of the flow environment the experiment is running on - env_name=WaveAttenuationPOEnv, - - # name of the network class the experiment is running on - network=RingNetwork, - - # simulator that is used by the experiment - simulator='traci', - - # sumo-related parameters (see flow.core.params.SumoParams) - sim=SumoParams( - sim_step=0.1, - render=False, - restart_instance=False - ), - - # environment related parameters (see flow.core.params.EnvParams) - env=EnvParams( - horizon=HORIZON, - warmup_steps=750, - clip_actions=False, - additional_params={ - "max_accel": 1, - "max_decel": 1, - "ring_length": [220, 270], - }, - ), - - # network-related parameters (see flow.core.params.NetParams and the - # network's documentation or ADDITIONAL_NET_PARAMS component) - net=NetParams( - additional_params={ - "length": 260, - "lanes": 1, - "speed_limit": 30, - "resolution": 40, - }, ), - - # vehicles to be placed in the network at the start of a rollout (see - # flow.core.params.VehicleParams) - veh=vehicles, - - # parameters specifying the positioning of vehicles upon initialization/ - # reset (see flow.core.params.InitialConfig) - initial=InitialConfig(), -) - - -def run_model(num_cpus=1, rollout_size=50, num_steps=50): - """Run the model for num_steps if provided. The total rollout length is rollout_size.""" - if num_cpus == 1: - constructor = env_constructor(params=flow_params, version=0)() - env = DummyVecEnv([lambda: constructor]) # The algorithms require a vectorized environment to run - else: - env = SubprocVecEnv([env_constructor(params=flow_params, version=i) for i in range(num_cpus)]) - - model = PPO2('MlpPolicy', env, verbose=1, n_steps=rollout_size) - model.learn(total_timesteps=num_steps) - return model - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--num_cpus', type=int, default=1, help='How many CPUs to use') - parser.add_argument('--num_steps', type=int, default=5000, help='How many total steps to perform learning over') - parser.add_argument('--rollout_size', type=int, default=1000, help='How many steps are in a training batch.') - parser.add_argument('--result_name', type=str, default='stabilize_ring', help='Name of saved model') - args = parser.parse_args() - model = run_model(args.num_cpus, args.rollout_size, args.num_steps) - # Save the model to a desired folder and then delete it to demonstrate loading - if not os.path.exists(os.path.realpath(os.path.expanduser('~/baseline_results'))): - os.makedirs(os.path.realpath(os.path.expanduser('~/baseline_results'))) - path = os.path.realpath(os.path.expanduser('~/baseline_results')) - save_path = os.path.join(path, args.result_name) - - print('Saving the trained model!') - model.save(save_path) - # dump the flow params - with open(os.path.join(path, args.result_name) + '.json', 'w') as outfile: - json.dump(flow_params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4) - del model - del flow_params - - # Replay the result by loading the model - print('Loading the trained model and testing it out!') - model = PPO2.load(save_path) - flow_params = get_flow_params(os.path.join(path, args.result_name) + '.json') - flow_params['sim'].render = True - env_constructor = env_constructor(params=flow_params, version=0)() - env = DummyVecEnv([lambda: env_constructor]) # The algorithms require a vectorized environment to run - obs = env.reset() - reward = 0 - for i in range(flow_params['env'].horizon): - action, _states = model.predict(obs) - obs, rewards, dones, info = env.step(action) - reward += rewards - print('the final reward is {}'.format(reward)) diff --git a/examples/stable_baselines/traffic_light_grid.py b/examples/stable_baselines/traffic_light_grid.py deleted file mode 100644 index 5419f0e21..000000000 --- a/examples/stable_baselines/traffic_light_grid.py +++ /dev/null @@ -1,301 +0,0 @@ -"""Traffic Light Grid example.""" - -import argparse -import json -import os - -from stable_baselines.common.vec_env import DummyVecEnv, SubprocVecEnv -from stable_baselines import PPO2 - -from flow.envs import TrafficLightGridPOEnv -from flow.networks import TrafficLightGridNetwork -from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams -from flow.core.params import SumoCarFollowingParams, InFlows -from flow.core.params import VehicleParams -from flow.controllers import SimCarFollowingController, GridRouter -from flow.utils.registry import env_constructor -from flow.utils.rllib import FlowParamsEncoder, get_flow_params - -# time horizon of a single rollout -HORIZON = 200 - - -def gen_edges(col_num, row_num): - """Generate the names of the outer edges in the traffic light grid network. - - Parameters - ---------- - col_num : int - number of columns in the traffic light grid - row_num : int - number of rows in the traffic light grid - - Returns - ------- - list of str - names of all the outer edges - """ - edges = [] - for i in range(col_num): - edges += ['left' + str(row_num) + '_' + str(i)] - edges += ['right' + '0' + '_' + str(i)] - - # build the left and then the right edgesØ - for i in range(row_num): - edges += ['bot' + str(i) + '_' + '0'] - edges += ['top' + str(i) + '_' + str(col_num)] - - return edges - - -def get_inflow_params(col_num, row_num, additional_net_params): - """Define the network and initial params in the presence of inflows. - - Parameters - ---------- - col_num : int - number of columns in the traffic light grid - row_num : int - number of rows in the traffic light grid - additional_net_params : dict - network-specific parameters that are unique to the traffic light grid - - Returns - ------- - flow.core.params.InitialConfig - parameters specifying the initial configuration of vehicles in the - network - flow.core.params.NetParams - network-specific parameters used to generate the network - """ - initial = InitialConfig( - spacing='custom', lanes_distribution=float('inf'), shuffle=True) - - inflow = InFlows() - outer_edges = gen_edges(col_num, row_num) - for i in range(len(outer_edges)): - inflow.add( - veh_type='idm', - edge=outer_edges[i], - probability=0.25, - departLane='free', - departSpeed=20) - - net = NetParams( - inflows=inflow, - additional_params=additional_net_params) - - return initial, net - - -def get_non_flow_params(enter_speed, add_net_params): - """Define the network and initial params in the absence of inflows. - - Note that when a vehicle leaves a network in this case, it is immediately - returns to the start of the row/column it was traversing, and in the same - direction as it was before. - - Parameters - ---------- - enter_speed : float - initial speed of vehicles as they enter the network. - add_net_params: dict - additional network-specific parameters (unique to the traffic light grid) - - Returns - ------- - flow.core.params.InitialConfig - parameters specifying the initial configuration of vehicles in the - network - flow.core.params.NetParams - network-specific parameters used to generate the network - """ - additional_init_params = {'enter_speed': enter_speed} - initial = InitialConfig( - spacing='custom', additional_params=additional_init_params) - net = NetParams( - additional_params=add_net_params) - - return initial, net - - -V_ENTER = 30 -INNER_LENGTH = 300 -LONG_LENGTH = 100 -SHORT_LENGTH = 300 -N_ROWS = 3 -N_COLUMNS = 3 -NUM_CARS_LEFT = 1 -NUM_CARS_RIGHT = 1 -NUM_CARS_TOP = 1 -NUM_CARS_BOT = 1 -tot_cars = (NUM_CARS_LEFT + NUM_CARS_RIGHT) * N_COLUMNS \ - + (NUM_CARS_BOT + NUM_CARS_TOP) * N_ROWS - -grid_array = { - "short_length": SHORT_LENGTH, - "inner_length": INNER_LENGTH, - "long_length": LONG_LENGTH, - "row_num": N_ROWS, - "col_num": N_COLUMNS, - "cars_left": NUM_CARS_LEFT, - "cars_right": NUM_CARS_RIGHT, - "cars_top": NUM_CARS_TOP, - "cars_bot": NUM_CARS_BOT -} - -additional_env_params = { - 'target_velocity': 50, - 'switch_time': 3.0, - 'num_observed': 2, - 'discrete': False, - 'tl_type': 'controlled' - } - -additional_net_params = { - 'speed_limit': 35, - 'grid_array': grid_array, - 'horizontal_lanes': 1, - 'vertical_lanes': 1 -} - -vehicles = VehicleParams() -vehicles.add( - veh_id='idm', - acceleration_controller=(SimCarFollowingController, {}), - car_following_params=SumoCarFollowingParams( - minGap=2.5, - decel=7.5, # avoid collisions at emergency stops - max_speed=V_ENTER, - speed_mode="all_checks", - ), - routing_controller=(GridRouter, {}), - num_vehicles=tot_cars) - -flow_params = dict( - # name of the experiment - exp_tag='traffic_light_grid', - - # name of the flow environment the experiment is running on - env_name=TrafficLightGridPOEnv, - - # name of the network class the experiment is running on - network=TrafficLightGridNetwork, - - # simulator that is used by the experiment - simulator='traci', - - # sumo-related parameters (see flow.core.params.SumoParams) - sim=SumoParams( - sim_step=1, - render=False, - ), - - # environment related parameters (see flow.core.params.EnvParams) - env=EnvParams( - horizon=HORIZON, - additional_params=additional_env_params, - ), - - # network-related parameters (see flow.core.params.NetParams and the - # network's documentation or ADDITIONAL_NET_PARAMS component). This is - # filled in by the setup_exps method below. - net=None, - - # vehicles to be placed in the network at the start of a rollout (see - # flow.core.params.VehicleParams) - veh=vehicles, - - # parameters specifying the positioning of vehicles upon initialization/ - # reset (see flow.core.params.InitialConfig). This is filled in by the - # setup_exps method below. - initial=None, -) - - -def setup_exps(use_inflows=False): - """Return the relevant components of an RLlib experiment. - - Parameters - ---------- - use_inflows : bool, optional - set to True if you would like to run the experiment with inflows of - vehicles from the edges, and False otherwise - - Returns - ------- - str - name of the training algorithm - str - name of the gym environment to be trained - dict - training configuration parameters - """ - # collect the initialization and network-specific parameters based on the - # choice to use inflows or not - if use_inflows: - initial_config, net_params = get_inflow_params( - col_num=N_COLUMNS, - row_num=N_ROWS, - additional_net_params=additional_net_params) - else: - initial_config, net_params = get_non_flow_params( - enter_speed=V_ENTER, - add_net_params=additional_net_params) - return initial_config, net_params - - -def run_model(num_cpus=1, rollout_size=50, num_steps=50, use_inflows=False): - """Run the model for num_steps if provided. The total rollout length is rollout_size.""" - initial_config, net_params = setup_exps(use_inflows) - # add the new parameters to flow_params - flow_params['initial'] = initial_config - flow_params['net'] = net_params - - if num_cpus == 1: - constructor = env_constructor(params=flow_params, version=0)() - env = DummyVecEnv([lambda: constructor]) # The algorithms require a vectorized environment to run - else: - env = SubprocVecEnv([env_constructor(params=flow_params, version=i) for i in range(num_cpus)]) - - model = PPO2('MlpPolicy', env, verbose=1, n_steps=rollout_size) - model.learn(total_timesteps=num_steps) - return model - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--num_cpus', type=int, default=1, help='How many CPUs to use') - parser.add_argument('--num_steps', type=int, default=5000, help='How many total steps to perform learning over') - parser.add_argument('--rollout_size', type=int, default=1000, help='How many steps are in a training batch.') - parser.add_argument('--result_name', type=str, default='traffic_light_grid', help='Name of saved model') - parser.add_argument('--use_inflows', action='store_true') - args = parser.parse_args() - model = run_model(args.num_cpus, args.rollout_size, args.num_steps, args.use_inflows) - # Save the model to a desired folder and then delete it to demonstrate loading - if not os.path.exists(os.path.realpath(os.path.expanduser('~/baseline_results'))): - os.makedirs(os.path.realpath(os.path.expanduser('~/baseline_results'))) - path = os.path.realpath(os.path.expanduser('~/baseline_results')) - save_path = os.path.join(path, args.result_name) - # dump the model - model.save(save_path) - # dump the flow params - with open(os.path.join(path, args.result_name) + '.json', 'w') as outfile: - json.dump(flow_params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4) - del model - del flow_params - - # Replay the result by loading the model - print('Loading the trained model and testing it out!') - model = PPO2.load(save_path) - flow_params = get_flow_params(os.path.join(path, args.result_name) + '.json') - flow_params['sim'].render = True - env_constructor = env_constructor(params=flow_params, version=0)() - env = DummyVecEnv([lambda: env_constructor]) # The algorithms require a vectorized environment to run - obs = env.reset() - reward = 0 - for i in range(flow_params['env'].horizon): - action, _states = model.predict(obs) - obs, rewards, dones, info = env.step(action) - reward += rewards - print('the final reward is {}'.format(reward)) diff --git a/examples/stable_baselines/velocity_bottleneck.py b/examples/stable_baselines/velocity_bottleneck.py deleted file mode 100644 index 413da8f7f..000000000 --- a/examples/stable_baselines/velocity_bottleneck.py +++ /dev/null @@ -1,211 +0,0 @@ -"""Bottleneck example. - -Bottleneck in which the actions are specifying a desired velocity -in a segment of space -""" -import argparse -import json -import os - -from stable_baselines.common.vec_env import DummyVecEnv, SubprocVecEnv -from stable_baselines import PPO2 - -from flow.envs import BottleneckDesiredVelocityEnv -from flow.networks import BottleneckNetwork -from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams, \ - InFlows, SumoCarFollowingParams, SumoLaneChangeParams -from flow.core.params import TrafficLightParams -from flow.core.params import VehicleParams -from flow.controllers import RLController, ContinuousRouter, \ - SimLaneChangeController -from flow.utils.registry import env_constructor -from flow.utils.rllib import FlowParamsEncoder, get_flow_params - -# time horizon of a single rollout -HORIZON = 1000 -# number of parallel workers -N_CPUS = 2 -# number of rollouts per training iteration -N_ROLLOUTS = N_CPUS * 4 - -SCALING = 1 -NUM_LANES = 4 * SCALING # number of lanes in the widest highway -DISABLE_TB = True -DISABLE_RAMP_METER = True -AV_FRAC = 0.10 - -vehicles = VehicleParams() -vehicles.add( - veh_id="human", - lane_change_controller=(SimLaneChangeController, {}), - routing_controller=(ContinuousRouter, {}), - car_following_params=SumoCarFollowingParams( - speed_mode="all_checks", - ), - lane_change_params=SumoLaneChangeParams( - lane_change_mode=0, - ), - num_vehicles=1 * SCALING) -vehicles.add( - veh_id="followerstopper", - acceleration_controller=(RLController, {}), - lane_change_controller=(SimLaneChangeController, {}), - routing_controller=(ContinuousRouter, {}), - car_following_params=SumoCarFollowingParams( - speed_mode=9, - ), - lane_change_params=SumoLaneChangeParams( - lane_change_mode=0, - ), - num_vehicles=1 * SCALING) - -controlled_segments = [("1", 1, False), ("2", 2, True), ("3", 2, True), - ("4", 2, True), ("5", 1, False)] -num_observed_segments = [("1", 1), ("2", 3), ("3", 3), ("4", 3), ("5", 1)] -additional_env_params = { - "target_velocity": 40, - "disable_tb": True, - "disable_ramp_metering": True, - "controlled_segments": controlled_segments, - "symmetric": False, - "observed_segments": num_observed_segments, - "reset_inflow": False, - "lane_change_duration": 5, - "max_accel": 3, - "max_decel": 3, - "inflow_range": [1000, 2000] -} - -# flow rate -flow_rate = 2300 * SCALING - -# percentage of flow coming out of each lane -inflow = InFlows() -inflow.add( - veh_type="human", - edge="1", - vehs_per_hour=flow_rate * (1 - AV_FRAC), - departLane="random", - departSpeed=10) -inflow.add( - veh_type="followerstopper", - edge="1", - vehs_per_hour=flow_rate * AV_FRAC, - departLane="random", - departSpeed=10) - -traffic_lights = TrafficLightParams() -if not DISABLE_TB: - traffic_lights.add(node_id="2") -if not DISABLE_RAMP_METER: - traffic_lights.add(node_id="3") - -additional_net_params = {"scaling": SCALING, "speed_limit": 23} -net_params = NetParams( - inflows=inflow, - additional_params=additional_net_params) - -flow_params = dict( - # name of the experiment - exp_tag="DesiredVelocity", - - # name of the flow environment the experiment is running on - env_name=BottleneckDesiredVelocityEnv, - - # name of the network class the experiment is running on - network=BottleneckNetwork, - - # simulator that is used by the experiment - simulator='traci', - - # sumo-related parameters (see flow.core.params.SumoParams) - sim=SumoParams( - sim_step=0.5, - render=False, - print_warnings=False, - restart_instance=True, - ), - - # environment related parameters (see flow.core.params.EnvParams) - env=EnvParams( - warmup_steps=40, - sims_per_step=1, - horizon=HORIZON, - additional_params=additional_env_params, - ), - - # network-related parameters (see flow.core.params.NetParams and the - # network's documentation or ADDITIONAL_NET_PARAMS component) - net=NetParams( - inflows=inflow, - additional_params=additional_net_params, - ), - - # vehicles to be placed in the network at the start of a rollout (see - # flow.core.params.VehicleParams) - veh=vehicles, - - # parameters specifying the positioning of vehicles upon initialization/ - # reset (see flow.core.params.InitialConfig) - initial=InitialConfig( - spacing="uniform", - min_gap=5, - lanes_distribution=float("inf"), - edges_distribution=["2", "3", "4", "5"], - ), - - # traffic lights to be introduced to specific nodes (see - # flow.core.params.TrafficLightParams) - tls=traffic_lights, -) - - -def run_model(num_cpus=1, rollout_size=50, num_steps=50): - """Run the model for num_steps if provided. The total rollout length is rollout_size.""" - if num_cpus == 1: - constructor = env_constructor(params=flow_params, version=0)() - env = DummyVecEnv([lambda: constructor]) # The algorithms require a vectorized environment to run - else: - env = SubprocVecEnv([env_constructor(params=flow_params, version=i) for i in range(num_cpus)]) - - model = PPO2('MlpPolicy', env, verbose=1, n_steps=rollout_size) - model.learn(total_timesteps=num_steps) - return model - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument('--num_cpus', type=int, default=1, help='How many CPUs to use') - parser.add_argument('--num_steps', type=int, default=5000, help='How many total steps to perform learning over') - parser.add_argument('--rollout_size', type=int, default=1000, help='How many steps are in a training batch.') - parser.add_argument('--result_name', type=str, default='velocity_bottleneck', help='Name of saved model') - args = parser.parse_args() - model = run_model(args.num_cpus, args.rollout_size, args.num_steps) - # Save the model to a desired folder and then delete it to demonstrate loading - if not os.path.exists(os.path.realpath(os.path.expanduser('~/baseline_results'))): - os.makedirs(os.path.realpath(os.path.expanduser('~/baseline_results'))) - path = os.path.realpath(os.path.expanduser('~/baseline_results')) - save_path = os.path.join(path, args.result_name) - - print('Saving the trained model!') - model.save(save_path) - # dump the flow params - with open(os.path.join(path, args.result_name) + '.json', 'w') as outfile: - json.dump(flow_params, outfile, cls=FlowParamsEncoder, sort_keys=True, indent=4) - del model - del flow_params - - # Replay the result by loading the model - print('Loading the trained model and testing it out!') - model = PPO2.load(save_path) - flow_params = get_flow_params(os.path.join(path, args.result_name) + '.json') - flow_params['sim'].render = True - env_constructor = env_constructor(params=flow_params, version=0)() - env = DummyVecEnv([lambda: env_constructor]) # The algorithms require a vectorized environment to run - obs = env.reset() - reward = 0 - for i in range(flow_params['env'].horizon): - action, _states = model.predict(obs) - obs, rewards, dones, info = env.step(action) - reward += rewards - print('the final reward is {}'.format(reward)) diff --git a/examples/sumo/__init__.py b/examples/sumo/__init__.py deleted file mode 100644 index d48602a8b..000000000 --- a/examples/sumo/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Empty init file to ensure that the examples documentation builds.""" diff --git a/examples/sumo/bay_bridge.py b/examples/sumo/bay_bridge.py deleted file mode 100644 index 49b4f7d23..000000000 --- a/examples/sumo/bay_bridge.py +++ /dev/null @@ -1,204 +0,0 @@ -"""Bay Bridge simulation.""" - -import os -import urllib.request - -from flow.core.params import SumoParams, EnvParams, NetParams, InitialConfig, \ - SumoCarFollowingParams, SumoLaneChangeParams, InFlows -from flow.core.params import VehicleParams -from flow.core.params import TrafficLightParams - -from flow.core.experiment import Experiment -from flow.envs.bay_bridge import BayBridgeEnv -from flow.networks.bay_bridge import BayBridgeNetwork, EDGES_DISTRIBUTION -from flow.controllers import SimCarFollowingController, BayBridgeRouter - -TEMPLATE = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "bay_bridge.net.xml") - - -def bay_bridge_example(render=None, - use_inflows=False, - use_traffic_lights=False): - """ - Perform a simulation of vehicles on the Oakland-San Francisco Bay Bridge. - - Parameters - ---------- - render: bool, optional - specifies whether to use the gui during execution - use_inflows: bool, optional - whether to activate inflows from the peripheries of the network - use_traffic_lights: bool, optional - whether to activate the traffic lights in the network - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles simulated by sumo on the Bay Bridge. - """ - sim_params = SumoParams(sim_step=0.6, overtake_right=True) - - if render is not None: - sim_params.render = render - - car_following_params = SumoCarFollowingParams( - speedDev=0.2, - speed_mode="all_checks", - ) - lane_change_params = SumoLaneChangeParams( - lc_assertive=20, - lc_pushy=0.8, - lc_speed_gain=4.0, - model="LC2013", - lane_change_mode="no_lat_collide", - # lcKeepRight=0.8 - ) - - vehicles = VehicleParams() - vehicles.add( - veh_id="human", - acceleration_controller=(SimCarFollowingController, {}), - routing_controller=(BayBridgeRouter, {}), - car_following_params=car_following_params, - lane_change_params=lane_change_params, - num_vehicles=1400) - - additional_env_params = {} - env_params = EnvParams(additional_params=additional_env_params) - - traffic_lights = TrafficLightParams() - - inflow = InFlows() - - if use_inflows: - # south - inflow.add( - veh_type="human", - edge="183343422", - vehsPerHour=528, - departLane="0", - departSpeed=20) - inflow.add( - veh_type="human", - edge="183343422", - vehsPerHour=864, - departLane="1", - departSpeed=20) - inflow.add( - veh_type="human", - edge="183343422", - vehsPerHour=600, - departLane="2", - departSpeed=20) - - inflow.add( - veh_type="human", - edge="393649534", - probability=0.1, - departLane="0", - departSpeed=20) # no data for this - - # west - inflow.add( - veh_type="human", - edge="11189946", - vehsPerHour=1752, - departLane="0", - departSpeed=20) - inflow.add( - veh_type="human", - edge="11189946", - vehsPerHour=2136, - departLane="1", - departSpeed=20) - inflow.add( - veh_type="human", - edge="11189946", - vehsPerHour=576, - departLane="2", - departSpeed=20) - - # north - inflow.add( - veh_type="human", - edge="28413687#0", - vehsPerHour=2880, - departLane="0", - departSpeed=20) - inflow.add( - veh_type="human", - edge="28413687#0", - vehsPerHour=2328, - departLane="1", - departSpeed=20) - inflow.add( - veh_type="human", - edge="28413687#0", - vehsPerHour=3060, - departLane="2", - departSpeed=20) - inflow.add( - veh_type="human", - edge="11198593", - probability=0.1, - departLane="0", - departSpeed=20) # no data for this - inflow.add( - veh_type="human", - edge="11197889", - probability=0.1, - departLane="0", - departSpeed=20) # no data for this - - # midway through bridge - inflow.add( - veh_type="human", - edge="35536683", - probability=0.1, - departLane="0", - departSpeed=20) # no data for this - - net_params = NetParams(inflows=inflow) - net_params.template = TEMPLATE - - # download the template from AWS - if use_traffic_lights: - my_url = "https://s3-us-west-1.amazonaws.com/flow.netfiles/" \ - "bay_bridge_TL_all_green.net.xml" - else: - my_url = "https://s3-us-west-1.amazonaws.com/flow.netfiles/" \ - "bay_bridge_junction_fix.net.xml" - my_file = urllib.request.urlopen(my_url) - data_to_write = my_file.read() - - with open( - os.path.join(os.path.dirname(os.path.abspath(__file__)), TEMPLATE), - "wb+") as f: - f.write(data_to_write) - - initial_config = InitialConfig( - spacing="uniform", - min_gap=15, - edges_distribution=EDGES_DISTRIBUTION.copy()) - - network = BayBridgeNetwork( - name="bay_bridge", - vehicles=vehicles, - traffic_lights=traffic_lights, - net_params=net_params, - initial_config=initial_config) - - env = BayBridgeEnv(env_params, sim_params, network) - - return Experiment(env) - - -if __name__ == "__main__": - # import the experiment variable - exp = bay_bridge_example( - render=True, use_inflows=False, use_traffic_lights=False) - - # run for a set number of rollouts / time steps - exp.run(1, 1500) diff --git a/examples/sumo/bay_bridge_toll.py b/examples/sumo/bay_bridge_toll.py deleted file mode 100644 index cd0aa110f..000000000 --- a/examples/sumo/bay_bridge_toll.py +++ /dev/null @@ -1,132 +0,0 @@ -"""Bay Bridge toll example.""" - -import os -import urllib.request - -from flow.core.params import SumoParams, EnvParams, NetParams, InitialConfig, \ - SumoLaneChangeParams, SumoCarFollowingParams, InFlows -from flow.core.params import VehicleParams - -from flow.core.experiment import Experiment -from flow.envs.bay_bridge import BayBridgeEnv - -from flow.networks.bay_bridge_toll import BayBridgeTollNetwork -from flow.networks.bay_bridge_toll import EDGES_DISTRIBUTION -from flow.controllers import SimCarFollowingController, BayBridgeRouter - -TEMPLATE = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "bottleneck.net.xml") - - -def bay_bridge_toll_example(render=None, use_traffic_lights=False): - """Perform a simulation of the toll portion of the Bay Bridge. - - This consists of the toll booth and sections of the road leading up to it. - - Parameters - ---------- - render : bool, optional - specifies whether to use the gui during execution - use_traffic_lights: bool, optional - whether to activate the traffic lights in the network - - Note - ---- - Unlike the bay_bridge_example, inflows are always activated here. - """ - sim_params = SumoParams(sim_step=0.4, overtake_right=True) - - if render is not None: - sim_params.render = render - - car_following_params = SumoCarFollowingParams( - speedDev=0.2, - speed_mode="all_checks", - ) - lane_change_params = SumoLaneChangeParams( - model="LC2013", - lcCooperative=0.2, - lcSpeedGain=15, - lane_change_mode="no_lat_collide", - ) - - vehicles = VehicleParams() - - vehicles.add( - veh_id="human", - acceleration_controller=(SimCarFollowingController, {}), - routing_controller=(BayBridgeRouter, {}), - car_following_params=car_following_params, - lane_change_params=lane_change_params, - num_vehicles=50) - - additional_env_params = {} - env_params = EnvParams(additional_params=additional_env_params) - - inflow = InFlows() - - inflow.add( - veh_type="human", - edge="393649534", - probability=0.2, - departLane="random", - departSpeed=10) - inflow.add( - veh_type="human", - edge="4757680", - probability=0.2, - departLane="random", - departSpeed=10) - inflow.add( - veh_type="human", - edge="32661316", - probability=0.2, - departLane="random", - departSpeed=10) - inflow.add( - veh_type="human", - edge="90077193#0", - vehs_per_hour=2000, - departLane="random", - departSpeed=10) - - net_params = NetParams(inflows=inflow, template=TEMPLATE) - - # download the template from AWS - if use_traffic_lights: - my_url = "https://s3-us-west-1.amazonaws.com/flow.netfiles/" \ - "bay_bridge_TL_all_green.net.xml" - else: - my_url = "https://s3-us-west-1.amazonaws.com/flow.netfiles/" \ - "bay_bridge_junction_fix.net.xml" - my_file = urllib.request.urlopen(my_url) - data_to_write = my_file.read() - - with open( - os.path.join(os.path.dirname(os.path.abspath(__file__)), TEMPLATE), - "wb+") as f: - f.write(data_to_write) - - initial_config = InitialConfig( - spacing="uniform", # "random", - min_gap=15, - edges_distribution=EDGES_DISTRIBUTION.copy()) - - network = BayBridgeTollNetwork( - name="bay_bridge_toll", - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config) - - env = BayBridgeEnv(env_params, sim_params, network) - - return Experiment(env) - - -if __name__ == "__main__": - # import the experiment variable - exp = bay_bridge_toll_example( - render=True, use_traffic_lights=False) - - # run for a set number of rollouts / time steps - exp.run(1, 1500) diff --git a/examples/sumo/bottlenecks.py b/examples/sumo/bottlenecks.py deleted file mode 100644 index 2d4c58c03..000000000 --- a/examples/sumo/bottlenecks.py +++ /dev/null @@ -1,202 +0,0 @@ -"""File demonstrating formation of congestion in bottleneck.""" - -from flow.core.params import SumoParams, EnvParams, NetParams, InitialConfig, \ - InFlows, SumoLaneChangeParams, SumoCarFollowingParams -from flow.core.params import VehicleParams -from flow.core.params import TrafficLightParams - -from flow.networks.bottleneck import BottleneckNetwork -from flow.controllers import SimLaneChangeController, ContinuousRouter -from flow.envs.bottleneck import BottleneckEnv -from flow.core.experiment import Experiment - -import logging - -import numpy as np -SCALING = 1 -DISABLE_TB = True - -# If set to False, ALINEA will control the ramp meter -DISABLE_RAMP_METER = True -INFLOW = 2300 - - -class BottleneckDensityExperiment(Experiment): - """Experiment object for bottleneck-specific simulations. - - Extends flow.core.experiment.Experiment - """ - - def __init__(self, env): - """Instantiate the bottleneck experiment.""" - super().__init__(env) - - def run(self, num_runs, num_steps, rl_actions=None, convert_to_csv=False): - """See parent class.""" - info_dict = {} - if rl_actions is None: - - def rl_actions(*_): - return None - - rets = [] - mean_rets = [] - ret_lists = [] - vels = [] - mean_vels = [] - std_vels = [] - mean_densities = [] - mean_outflows = [] - for i in range(num_runs): - vel = np.zeros(num_steps) - logging.info('Iter #' + str(i)) - ret = 0 - ret_list = [] - step_outflows = [] - step_densities = [] - state = self.env.reset() - for j in range(num_steps): - state, reward, done, _ = self.env.step(rl_actions(state)) - vel[j] = np.mean(self.env.k.vehicle.get_speed( - self.env.k.vehicle.get_ids())) - ret += reward - ret_list.append(reward) - - env = self.env - step_outflow = env.k.vehicle.get_outflow_rate(20) - density = self.env.get_bottleneck_density() - - step_outflows.append(step_outflow) - step_densities.append(density) - if done: - break - rets.append(ret) - vels.append(vel) - mean_densities.append(sum(step_densities[100:]) / - (num_steps - 100)) - env = self.env - outflow = env.k.vehicle.get_outflow_rate(10000) - mean_outflows.append(outflow) - mean_rets.append(np.mean(ret_list)) - ret_lists.append(ret_list) - mean_vels.append(np.mean(vel)) - std_vels.append(np.std(vel)) - print('Round {0}, return: {1}'.format(i, ret)) - - info_dict['returns'] = rets - info_dict['velocities'] = vels - info_dict['mean_returns'] = mean_rets - info_dict['per_step_returns'] = ret_lists - info_dict['average_outflow'] = np.mean(mean_outflows) - info_dict['per_rollout_outflows'] = mean_outflows - - info_dict['average_rollout_density_outflow'] = np.mean(mean_densities) - - print('Average, std return: {}, {}'.format( - np.mean(rets), np.std(rets))) - print('Average, std speed: {}, {}'.format( - np.mean(mean_vels), np.std(std_vels))) - self.env.terminate() - - return info_dict - - -def bottleneck_example(flow_rate, horizon, restart_instance=False, - render=None): - """ - Perform a simulation of vehicles on a bottleneck. - - Parameters - ---------- - flow_rate : float - total inflow rate of vehicles into the bottleneck - horizon : int - time horizon - restart_instance: bool, optional - whether to restart the instance upon reset - render: bool, optional - specifies whether to use the gui during execution - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles on a bottleneck. - """ - if render is None: - render = False - - sim_params = SumoParams( - sim_step=0.5, - render=render, - overtake_right=False, - restart_instance=restart_instance) - - vehicles = VehicleParams() - - vehicles.add( - veh_id="human", - lane_change_controller=(SimLaneChangeController, {}), - routing_controller=(ContinuousRouter, {}), - car_following_params=SumoCarFollowingParams( - speed_mode=25, - ), - lane_change_params=SumoLaneChangeParams( - lane_change_mode=1621, - ), - num_vehicles=1) - - additional_env_params = { - "target_velocity": 40, - "max_accel": 1, - "max_decel": 1, - "lane_change_duration": 5, - "add_rl_if_exit": False, - "disable_tb": DISABLE_TB, - "disable_ramp_metering": DISABLE_RAMP_METER - } - env_params = EnvParams( - horizon=horizon, additional_params=additional_env_params) - - inflow = InFlows() - inflow.add( - veh_type="human", - edge="1", - vehsPerHour=flow_rate, - departLane="random", - departSpeed=10) - - traffic_lights = TrafficLightParams() - if not DISABLE_TB: - traffic_lights.add(node_id="2") - if not DISABLE_RAMP_METER: - traffic_lights.add(node_id="3") - - additional_net_params = {"scaling": SCALING, "speed_limit": 23} - net_params = NetParams( - inflows=inflow, - additional_params=additional_net_params) - - initial_config = InitialConfig( - spacing="random", - min_gap=5, - lanes_distribution=float("inf"), - edges_distribution=["2", "3", "4", "5"]) - - network = BottleneckNetwork( - name="bay_bridge_toll", - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config, - traffic_lights=traffic_lights) - - env = BottleneckEnv(env_params, sim_params, network) - - return BottleneckDensityExperiment(env) - - -if __name__ == '__main__': - # import the experiment variable - # inflow, number of steps, binary - exp = bottleneck_example(INFLOW, 1000, render=True) - exp.run(5, 1000) diff --git a/examples/sumo/figure_eight.py b/examples/sumo/figure_eight.py deleted file mode 100755 index e7cb23626..000000000 --- a/examples/sumo/figure_eight.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Example of a figure 8 network with human-driven vehicles. - -Right-of-way dynamics near the intersection causes vehicles to queue up on -either side of the intersection, leading to a significant reduction in the -average speed of vehicles in the network. -""" -from flow.controllers import IDMController, StaticLaneChanger, ContinuousRouter -from flow.core.experiment import Experiment -from flow.core.params import SumoParams, EnvParams, NetParams, \ - SumoCarFollowingParams -from flow.core.params import VehicleParams -from flow.envs.ring.accel import AccelEnv, ADDITIONAL_ENV_PARAMS -from flow.networks.figure_eight import FigureEightNetwork, ADDITIONAL_NET_PARAMS - - -def figure_eight_example(render=None): - """ - Perform a simulation of vehicles on a figure eight. - - Parameters - ---------- - render: bool, optional - specifies whether to use the gui during execution - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles on a figure eight. - """ - sim_params = SumoParams(render=True) - - if render is not None: - sim_params.render = render - - vehicles = VehicleParams() - vehicles.add( - veh_id="idm", - acceleration_controller=(IDMController, {}), - lane_change_controller=(StaticLaneChanger, {}), - routing_controller=(ContinuousRouter, {}), - car_following_params=SumoCarFollowingParams( - speed_mode="obey_safe_speed", - decel=1.5, - ), - initial_speed=0, - num_vehicles=14) - - env_params = EnvParams(additional_params=ADDITIONAL_ENV_PARAMS) - - additional_net_params = ADDITIONAL_NET_PARAMS.copy() - net_params = NetParams(additional_params=additional_net_params) - - network = FigureEightNetwork( - name="FigureEight", - vehicles=vehicles, - net_params=net_params) - - env = AccelEnv(env_params, sim_params, network) - - return Experiment(env) - - -if __name__ == "__main__": - # import the experiment variable - exp = figure_eight_example() - - # run for a set number of rollouts / time steps - exp.run(1, 1500) diff --git a/examples/sumo/highway.py b/examples/sumo/highway.py deleted file mode 100644 index 5a41df32b..000000000 --- a/examples/sumo/highway.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Example of an open multi-lane network with human-driven vehicles.""" - -from flow.controllers import IDMController -from flow.core.experiment import Experiment -from flow.core.params import SumoParams, EnvParams, \ - NetParams, InitialConfig, InFlows -from flow.core.params import VehicleParams -from flow.envs.ring.lane_change_accel import LaneChangeAccelEnv, \ - ADDITIONAL_ENV_PARAMS -from flow.networks.highway import HighwayNetwork, ADDITIONAL_NET_PARAMS - - -def highway_example(render=None): - """ - Perform a simulation of vehicles on a highway. - - Parameters - ---------- - render : bool, optional - specifies whether to use the gui during execution - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles on a figure eight. - """ - sim_params = SumoParams(render=True) - - if render is not None: - sim_params.render = render - - vehicles = VehicleParams() - vehicles.add( - veh_id="human", - acceleration_controller=(IDMController, {}), - num_vehicles=20) - vehicles.add( - veh_id="human2", - acceleration_controller=(IDMController, {}), - num_vehicles=20) - - env_params = EnvParams(additional_params=ADDITIONAL_ENV_PARAMS) - - inflow = InFlows() - inflow.add( - veh_type="human", - edge="highway_0", - probability=0.25, - departLane="free", - departSpeed=20) - inflow.add( - veh_type="human2", - edge="highway_0", - probability=0.25, - departLane="free", - departSpeed=20) - - additional_net_params = ADDITIONAL_NET_PARAMS.copy() - net_params = NetParams( - inflows=inflow, additional_params=additional_net_params) - - initial_config = InitialConfig(spacing="uniform", shuffle=True) - - network = HighwayNetwork( - name="highway", - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config) - - env = LaneChangeAccelEnv(env_params, sim_params, network) - - return Experiment(env) - - -if __name__ == "__main__": - # import the experiment variable - exp = highway_example() - - # run for a set number of rollouts / time steps - exp.run(1, 1500) diff --git a/examples/sumo/highway_ramps.py b/examples/sumo/highway_ramps.py deleted file mode 100644 index f6b3ef004..000000000 --- a/examples/sumo/highway_ramps.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Example of a highway section network with on/off ramps.""" - -from flow.core.params import SumoParams, EnvParams, \ - NetParams, InitialConfig, InFlows, SumoCarFollowingParams, \ - SumoLaneChangeParams -from flow.core.params import VehicleParams -from flow.core.experiment import Experiment -from flow.networks.highway_ramps import HighwayRampsNetwork, \ - ADDITIONAL_NET_PARAMS -from flow.envs.ring.accel import AccelEnv, ADDITIONAL_ENV_PARAMS - - -additional_net_params = ADDITIONAL_NET_PARAMS.copy() - -# lengths -additional_net_params["highway_length"] = 1200 -additional_net_params["on_ramps_length"] = 200 -additional_net_params["off_ramps_length"] = 200 - -# number of lanes -additional_net_params["highway_lanes"] = 3 -additional_net_params["on_ramps_lanes"] = 1 -additional_net_params["off_ramps_lanes"] = 1 - -# speed limits -additional_net_params["highway_speed"] = 30 -additional_net_params["on_ramps_speed"] = 20 -additional_net_params["off_ramps_speed"] = 20 - -# ramps -additional_net_params["on_ramps_pos"] = [400] -additional_net_params["off_ramps_pos"] = [800] - -# probability of exiting at the next off-ramp -additional_net_params["next_off_ramp_proba"] = 0.25 - -# inflow rates in vehs/hour -HIGHWAY_INFLOW_RATE = 4000 -ON_RAMPS_INFLOW_RATE = 350 - - -def highway_ramps_example(render=None): - """ - Perform a simulation of vehicles on a highway section with ramps. - - Parameters - ---------- - render: bool, optional - Specifies whether or not to use the GUI during the simulation. - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-RL experiment demonstrating the performance of human-driven - vehicles on a highway section with on and off ramps. - """ - sim_params = SumoParams( - render=True, - emission_path="./data/", - sim_step=0.2, - restart_instance=True) - - if render is not None: - sim_params.render = render - - vehicles = VehicleParams() - vehicles.add( - veh_id="human", - car_following_params=SumoCarFollowingParams( - speed_mode="obey_safe_speed", # for safer behavior at the merges - tau=1.5 # larger distance between cars - ), - lane_change_params=SumoLaneChangeParams(lane_change_mode=1621)) - - env_params = EnvParams( - additional_params=ADDITIONAL_ENV_PARAMS, - sims_per_step=5, - warmup_steps=0) - - inflows = InFlows() - inflows.add( - veh_type="human", - edge="highway_0", - vehs_per_hour=HIGHWAY_INFLOW_RATE, - depart_lane="free", - depart_speed="max", - name="highway_flow") - for i in range(len(additional_net_params["on_ramps_pos"])): - inflows.add( - veh_type="human", - edge="on_ramp_{}".format(i), - vehs_per_hour=ON_RAMPS_INFLOW_RATE, - depart_lane="first", - depart_speed="max", - name="on_ramp_flow") - - net_params = NetParams( - inflows=inflows, - additional_params=additional_net_params) - - initial_config = InitialConfig() # no vehicles initially - - network = HighwayRampsNetwork( - name="highway-ramp", - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config) - - env = AccelEnv(env_params, sim_params, network) - - return Experiment(env) - - -if __name__ == "__main__": - exp = highway_ramps_example() - exp.run(1, 3600, convert_to_csv=True) diff --git a/examples/sumo/merge.py b/examples/sumo/merge.py deleted file mode 100644 index 943662998..000000000 --- a/examples/sumo/merge.py +++ /dev/null @@ -1,100 +0,0 @@ -"""Example of a merge network with human-driven vehicles. - -In the absence of autonomous vehicles, the network exhibits properties of -convective instability, with perturbations propagating upstream from the merge -point before exiting the network. -""" - -from flow.core.params import SumoParams, EnvParams, \ - NetParams, InitialConfig, InFlows, SumoCarFollowingParams -from flow.core.params import VehicleParams -from flow.core.experiment import Experiment -from flow.networks.merge import MergeNetwork, ADDITIONAL_NET_PARAMS -from flow.controllers import IDMController -from flow.envs.merge import MergePOEnv, ADDITIONAL_ENV_PARAMS - -# inflow rate at the highway -FLOW_RATE = 2000 - - -def merge_example(render=None): - """ - Perform a simulation of vehicles on a merge. - - Parameters - ---------- - render: bool, optional - specifies whether to use the gui during execution - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles on a merge. - """ - sim_params = SumoParams( - render=True, - emission_path="./data/", - sim_step=0.2, - restart_instance=False) - - if render is not None: - sim_params.render = render - - vehicles = VehicleParams() - vehicles.add( - veh_id="human", - acceleration_controller=(IDMController, { - "noise": 0.2 - }), - car_following_params=SumoCarFollowingParams( - speed_mode="obey_safe_speed", - ), - num_vehicles=5) - - env_params = EnvParams( - additional_params=ADDITIONAL_ENV_PARAMS, - sims_per_step=5, - warmup_steps=0) - - inflow = InFlows() - inflow.add( - veh_type="human", - edge="inflow_highway", - vehs_per_hour=FLOW_RATE, - departLane="free", - departSpeed=10) - inflow.add( - veh_type="human", - edge="inflow_merge", - vehs_per_hour=100, - departLane="free", - departSpeed=7.5) - - additional_net_params = ADDITIONAL_NET_PARAMS.copy() - additional_net_params["merge_lanes"] = 1 - additional_net_params["highway_lanes"] = 1 - additional_net_params["pre_merge_length"] = 500 - net_params = NetParams( - inflows=inflow, - additional_params=additional_net_params) - - initial_config = InitialConfig(spacing="uniform", perturbation=5.0) - - network = MergeNetwork( - name="merge-baseline", - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config) - - env = MergePOEnv(env_params, sim_params, network) - - return Experiment(env) - - -if __name__ == "__main__": - # import the experiment variable - exp = merge_example() - - # run for a set number of rollouts / time steps - exp.run(1, 3600, convert_to_csv=False) diff --git a/examples/sumo/minicity.py b/examples/sumo/minicity.py deleted file mode 100644 index 2f4b72741..000000000 --- a/examples/sumo/minicity.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Example of modified minicity network with human-driven vehicles.""" -from flow.controllers import IDMController -from flow.controllers import RLController -from flow.core.experiment import Experiment -from flow.core.params import SumoParams, EnvParams, NetParams, InitialConfig -from flow.core.params import SumoCarFollowingParams, SumoLaneChangeParams -from flow.core.params import VehicleParams -from flow.envs.ring.accel import AccelEnv, ADDITIONAL_ENV_PARAMS -from flow.networks.minicity import MiniCityNetwork -from flow.controllers.routing_controllers import MinicityRouter -import numpy as np - -np.random.seed(204) - - -def minicity_example(render=None, - save_render=None, - sight_radius=None, - pxpm=None, - show_radius=None): - """Perform a simulation of modified minicity of University of Delaware. - - Parameters - ---------- - render: bool, optional - specifies whether to use the gui during execution - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles on the minicity network. - """ - sim_params = SumoParams(sim_step=0.25) - - # update sim_params values if provided as inputs - sim_params.render = render or sim_params.render - sim_params.save_render = save_render or sim_params.save_render - sim_params.sight_radius = sight_radius or sim_params.sight_radius - sim_params.pxpm = pxpm or sim_params.pxpm - sim_params.show_radius = show_radius or sim_params.show_radius - - vehicles = VehicleParams() - vehicles.add( - veh_id="idm", - acceleration_controller=(IDMController, {}), - routing_controller=(MinicityRouter, {}), - car_following_params=SumoCarFollowingParams( - speed_mode=1, - ), - lane_change_params=SumoLaneChangeParams( - lane_change_mode="no_lat_collide", - ), - initial_speed=0, - num_vehicles=90) - vehicles.add( - veh_id="rl", - acceleration_controller=(RLController, {}), - routing_controller=(MinicityRouter, {}), - car_following_params=SumoCarFollowingParams( - speed_mode="obey_safe_speed", - ), - initial_speed=0, - num_vehicles=10) - - env_params = EnvParams(additional_params=ADDITIONAL_ENV_PARAMS) - - net_params = NetParams() - - initial_config = InitialConfig( - spacing="random", - min_gap=5 - ) - network = MiniCityNetwork( - name="minicity", - vehicles=vehicles, - initial_config=initial_config, - net_params=net_params) - - env = AccelEnv(env_params, sim_params, network) - - return Experiment(env) - - -if __name__ == "__main__": - # import the experiment variable - # There are six modes of pyglet rendering: - # No rendering: minicity_example(render=False) - # SUMO-GUI rendering: minicity_example(render=True) - # Static grayscale rendering: minicity_example(render="gray") - # Dynamic grayscale rendering: minicity_example(render="dgray") - # Static RGB rendering: minicity_example(render="rgb") - # Dynamic RGB rendering: minicity_example(render="drgb") - exp = minicity_example(render='drgb', - save_render=False, - sight_radius=30, - pxpm=3, - show_radius=True) - - # run for a set number of rollouts / time steps - exp.run(1, 750) diff --git a/examples/sumo/sugiyama.py b/examples/sumo/sugiyama.py deleted file mode 100755 index a2c03d060..000000000 --- a/examples/sumo/sugiyama.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Used as an example of sugiyama experiment. - -This example consists of 22 IDM cars on a ring creating shockwaves. -""" - -from flow.controllers import IDMController, ContinuousRouter -from flow.core.experiment import Experiment -from flow.core.params import SumoParams, EnvParams, \ - InitialConfig, NetParams, SumoCarFollowingParams -from flow.core.params import VehicleParams -from flow.envs.ring.accel import AccelEnv, ADDITIONAL_ENV_PARAMS -from flow.networks.ring import RingNetwork, ADDITIONAL_NET_PARAMS - - -def sugiyama_example(render=None): - """ - Perform a simulation of vehicles on a ring road. - - Parameters - ---------- - render : bool, optional - specifies whether to use the gui during execution - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles on a ring road. - """ - sim_params = SumoParams(sim_step=0.1, render=True) - - if render is not None: - sim_params.render = render - - vehicles = VehicleParams() - vehicles.add( - veh_id="idm", - acceleration_controller=(IDMController, {}), - car_following_params=SumoCarFollowingParams( - min_gap=0 - ), - routing_controller=(ContinuousRouter, {}), - num_vehicles=22) - - env_params = EnvParams(additional_params=ADDITIONAL_ENV_PARAMS) - - additional_net_params = ADDITIONAL_NET_PARAMS.copy() - net_params = NetParams(additional_params=additional_net_params) - - initial_config = InitialConfig(bunching=20) - - network = RingNetwork( - name="sugiyama", - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config) - - env = AccelEnv(env_params, sim_params, network) - - return Experiment(env) - - -if __name__ == "__main__": - # import the experiment variable - exp = sugiyama_example() - - # run for a set number of rollouts / time steps - exp.run(1, 1500) diff --git a/examples/sumo/traffic_light_grid.py b/examples/sumo/traffic_light_grid.py deleted file mode 100644 index 5af8347ea..000000000 --- a/examples/sumo/traffic_light_grid.py +++ /dev/null @@ -1,233 +0,0 @@ -"""Traffic Light Grid example.""" -from flow.controllers import GridRouter -from flow.core.experiment import Experiment -from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams -from flow.core.params import VehicleParams -from flow.core.params import TrafficLightParams -from flow.core.params import SumoCarFollowingParams -from flow.core.params import InFlows -from flow.envs.ring.accel import AccelEnv, ADDITIONAL_ENV_PARAMS -from flow.networks import TrafficLightGridNetwork - - -def gen_edges(col_num, row_num): - """Generate the names of the outer edges in the traffic light grid network. - - Parameters - ---------- - col_num : int - number of columns in the traffic light grid - row_num : int - number of rows in the traffic light grid - - Returns - ------- - list of str - names of all the outer edges - """ - edges = [] - - # build the left and then the right edges - for i in range(col_num): - edges += ['left' + str(row_num) + '_' + str(i)] - edges += ['right' + '0' + '_' + str(i)] - - # build the bottom and then top edges - for i in range(row_num): - edges += ['bot' + str(i) + '_' + '0'] - edges += ['top' + str(i) + '_' + str(col_num)] - - return edges - - -def get_flow_params(col_num, row_num, additional_net_params): - """Define the network and initial params in the presence of inflows. - - Parameters - ---------- - col_num : int - number of columns in the traffic light grid - row_num : int - number of rows in the traffic light grid - additional_net_params : dict - network-specific parameters that are unique to the traffic light grid - - Returns - ------- - flow.core.params.InitialConfig - parameters specifying the initial configuration of vehicles in the - network - flow.core.params.NetParams - network-specific parameters used to generate the network - """ - initial = InitialConfig( - spacing='custom', lanes_distribution=float('inf'), shuffle=True) - - inflow = InFlows() - outer_edges = gen_edges(col_num, row_num) - for i in range(len(outer_edges)): - inflow.add( - veh_type='human', - edge=outer_edges[i], - probability=0.25, - departLane='free', - departSpeed=20) - - net = NetParams( - inflows=inflow, - additional_params=additional_net_params) - - return initial, net - - -def get_non_flow_params(enter_speed, add_net_params): - """Define the network and initial params in the absence of inflows. - - Note that when a vehicle leaves a network in this case, it is immediately - returns to the start of the row/column it was traversing, and in the same - direction as it was before. - - Parameters - ---------- - enter_speed : float - initial speed of vehicles as they enter the network. - add_net_params: dict - additional network-specific parameters (unique to the traffic light grid) - - Returns - ------- - flow.core.params.InitialConfig - parameters specifying the initial configuration of vehicles in the - network - flow.core.params.NetParams - network-specific parameters used to generate the network - """ - additional_init_params = {'enter_speed': enter_speed} - initial = InitialConfig( - spacing='custom', additional_params=additional_init_params) - net = NetParams(additional_params=add_net_params) - - return initial, net - - -def traffic_light_grid_example(render=None, use_inflows=False): - """ - Perform a simulation of vehicles on a traffic light grid. - - Parameters - ---------- - render: bool, optional - specifies whether to use the gui during execution - use_inflows : bool, optional - set to True if you would like to run the experiment with inflows of - vehicles from the edges, and False otherwise - - Returns - ------- - exp: flow.core.experiment.Experiment - A non-rl experiment demonstrating the performance of human-driven - vehicles and balanced traffic lights on a traffic light grid. - """ - v_enter = 10 - inner_length = 300 - long_length = 500 - short_length = 300 - n_rows = 2 - n_columns = 3 - num_cars_left = 20 - num_cars_right = 20 - num_cars_top = 20 - num_cars_bot = 20 - tot_cars = (num_cars_left + num_cars_right) * n_columns \ - + (num_cars_top + num_cars_bot) * n_rows - - grid_array = { - "short_length": short_length, - "inner_length": inner_length, - "long_length": long_length, - "row_num": n_rows, - "col_num": n_columns, - "cars_left": num_cars_left, - "cars_right": num_cars_right, - "cars_top": num_cars_top, - "cars_bot": num_cars_bot - } - - sim_params = SumoParams(sim_step=0.1, render=True) - - if render is not None: - sim_params.render = render - - vehicles = VehicleParams() - vehicles.add( - veh_id="human", - routing_controller=(GridRouter, {}), - car_following_params=SumoCarFollowingParams( - min_gap=2.5, - decel=7.5, # avoid collisions at emergency stops - ), - num_vehicles=tot_cars) - - env_params = EnvParams(additional_params=ADDITIONAL_ENV_PARAMS) - - tl_logic = TrafficLightParams(baseline=False) - phases = [{ - "duration": "31", - "minDur": "8", - "maxDur": "45", - "state": "GrGrGrGrGrGr" - }, { - "duration": "6", - "minDur": "3", - "maxDur": "6", - "state": "yryryryryryr" - }, { - "duration": "31", - "minDur": "8", - "maxDur": "45", - "state": "rGrGrGrGrGrG" - }, { - "duration": "6", - "minDur": "3", - "maxDur": "6", - "state": "ryryryryryry" - }] - tl_logic.add("center0", phases=phases, programID=1) - tl_logic.add("center1", phases=phases, programID=1) - tl_logic.add("center2", phases=phases, programID=1, tls_type="actuated") - - additional_net_params = { - "grid_array": grid_array, - "speed_limit": 35, - "horizontal_lanes": 1, - "vertical_lanes": 1 - } - - if use_inflows: - initial_config, net_params = get_flow_params( - col_num=n_columns, - row_num=n_rows, - additional_net_params=additional_net_params) - else: - initial_config, net_params = get_non_flow_params( - enter_speed=v_enter, - add_net_params=additional_net_params) - - network = TrafficLightGridNetwork( - name="grid-intersection", - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config, - traffic_lights=tl_logic) - - env = AccelEnv(env_params, sim_params, network) - - return Experiment(env) - - -if __name__ == "__main__": - # import the experiment variable - exp = traffic_light_grid_example() - - # run for a set number of rollouts / time steps - exp.run(1, 1500) diff --git a/examples/train.py b/examples/train.py new file mode 100644 index 000000000..5f8edbb22 --- /dev/null +++ b/examples/train.py @@ -0,0 +1,375 @@ +"""Runner script for single and multi-agent reinforcement learning experiments. + +This script performs an RL experiment using the PPO algorithm. Choice of +hyperparameters can be seen and adjusted from the code below. + +Usage + python train.py EXP_CONFIG +""" +import argparse +import json +import os +import sys +from time import strftime +from copy import deepcopy + +from flow.core.util import ensure_dir +from flow.utils.registry import env_constructor +from flow.utils.rllib import FlowParamsEncoder, get_flow_params +from flow.utils.registry import make_create_env + + +def parse_args(args): + """Parse training options user can specify in command line. + + Returns + ------- + argparse.Namespace + the output parser object + """ + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description="Parse argument used when running a Flow simulation.", + epilog="python train.py EXP_CONFIG") + + # required input parameters + parser.add_argument( + 'exp_config', type=str, + help='Name of the experiment configuration file, as located in ' + 'exp_configs/rl/singleagent or exp_configs/rl/multiagent.') + + # optional input parameters + parser.add_argument( + '--rl_trainer', type=str, default="rllib", + help='the RL trainer to use. either rllib or Stable-Baselines') + + parser.add_argument( + '--num_cpus', type=int, default=1, + help='How many CPUs to use') + parser.add_argument( + '--num_steps', type=int, default=5000, + help='How many total steps to perform learning over') + parser.add_argument( + '--rollout_size', type=int, default=1000, + help='How many steps are in a training batch.') + parser.add_argument( + '--checkpoint_path', type=str, default=None, + help='Directory with checkpoint to restore training from.') + + return parser.parse_known_args(args)[0] + + +def run_model_stablebaseline(flow_params, + num_cpus=1, + rollout_size=50, + num_steps=50): + """Run the model for num_steps if provided. + + Parameters + ---------- + flow_params : dict + flow-specific parameters + num_cpus : int + number of CPUs used during training + rollout_size : int + length of a single rollout + num_steps : int + total number of training steps + The total rollout length is rollout_size. + + Returns + ------- + stable_baselines.* + the trained model + """ + from stable_baselines.common.vec_env import DummyVecEnv, SubprocVecEnv + from stable_baselines import PPO2 + + if num_cpus == 1: + constructor = env_constructor(params=flow_params, version=0)() + # The algorithms require a vectorized environment to run + env = DummyVecEnv([lambda: constructor]) + else: + env = SubprocVecEnv([env_constructor(params=flow_params, version=i) + for i in range(num_cpus)]) + + train_model = PPO2('MlpPolicy', env, verbose=1, n_steps=rollout_size) + train_model.learn(total_timesteps=num_steps) + return train_model + + +def setup_exps_rllib(flow_params, + n_cpus, + n_rollouts, + policy_graphs=None, + policy_mapping_fn=None, + policies_to_train=None): + """Return the relevant components of an RLlib experiment. + + Parameters + ---------- + flow_params : dict + flow-specific parameters (see flow/utils/registry.py) + n_cpus : int + number of CPUs to run the experiment over + n_rollouts : int + number of rollouts per training iteration + policy_graphs : dict, optional + TODO + policy_mapping_fn : function, optional + TODO + policies_to_train : list of str, optional + TODO + + Returns + ------- + str + name of the training algorithm + str + name of the gym environment to be trained + dict + training configuration parameters + """ + from ray import tune + from ray.tune.registry import register_env + try: + from ray.rllib.agents.agent import get_agent_class + except ImportError: + from ray.rllib.agents.registry import get_agent_class + + horizon = flow_params['env'].horizon + + alg_run = "PPO" + + agent_cls = get_agent_class(alg_run) + config = deepcopy(agent_cls._default_config) + + config["num_workers"] = n_cpus + config["train_batch_size"] = horizon * n_rollouts + config["gamma"] = 0.999 # discount rate + config["model"].update({"fcnet_hiddens": [32, 32, 32]}) + config["use_gae"] = True + config["lambda"] = 0.97 + config["kl_target"] = 0.02 + config["num_sgd_iter"] = 10 + config["horizon"] = horizon + + # save the flow params for replay + flow_json = json.dumps( + flow_params, cls=FlowParamsEncoder, sort_keys=True, indent=4) + config['env_config']['flow_params'] = flow_json + config['env_config']['run'] = alg_run + + # multiagent configuration + if policy_graphs is not None: + print("policy_graphs", policy_graphs) + config['multiagent'].update({'policies': policy_graphs}) + if policy_mapping_fn is not None: + config['multiagent'].update( + {'policy_mapping_fn': tune.function(policy_mapping_fn)}) + if policies_to_train is not None: + config['multiagent'].update({'policies_to_train': policies_to_train}) + + create_env, gym_name = make_create_env(params=flow_params) + + # Register as rllib env + register_env(gym_name, create_env) + return alg_run, gym_name, config + + +def train_rllib(submodule, flags): + """Train policies using the PPO algorithm in RLlib.""" + import ray + from ray.tune import run_experiments + + flow_params = submodule.flow_params + n_cpus = submodule.N_CPUS + n_rollouts = submodule.N_ROLLOUTS + policy_graphs = getattr(submodule, "POLICY_GRAPHS", None) + policy_mapping_fn = getattr(submodule, "policy_mapping_fn", None) + policies_to_train = getattr(submodule, "policies_to_train", None) + + alg_run, gym_name, config = setup_exps_rllib( + flow_params, n_cpus, n_rollouts, + policy_graphs, policy_mapping_fn, policies_to_train) + + ray.init(num_cpus=n_cpus + 1, object_store_memory=200 * 1024 * 1024) + exp_config = { + "run": alg_run, + "env": gym_name, + "config": { + **config + }, + "checkpoint_freq": 20, + "checkpoint_at_end": True, + "max_failures": 999, + "stop": { + "training_iteration": flags.num_steps, + }, + } + + if flags.checkpoint_path is not None: + exp_config['restore'] = flags.checkpoint_path + run_experiments({flow_params["exp_tag"]: exp_config}) + + +def train_h_baselines(env_name, args, multiagent): + """Train policies using SAC and TD3 with h-baselines.""" + from hbaselines.algorithms import OffPolicyRLAlgorithm + from hbaselines.utils.train import parse_options, get_hyperparameters + + # Get the command-line arguments that are relevant here + args = parse_options(description="", example_usage="", args=args) + + # the base directory that the logged data will be stored in + base_dir = "training_data" + + for i in range(args.n_training): + # value of the next seed + seed = args.seed + i + + # The time when the current experiment started. + now = strftime("%Y-%m-%d-%H:%M:%S") + + # Create a save directory folder (if it doesn't exist). + dir_name = os.path.join(base_dir, '{}/{}'.format(args.env_name, now)) + ensure_dir(dir_name) + + # Get the policy class. + if args.alg == "TD3": + if multiagent: + from hbaselines.multi_fcnet.td3 import MultiFeedForwardPolicy + policy = MultiFeedForwardPolicy + else: + from hbaselines.fcnet.td3 import FeedForwardPolicy + policy = FeedForwardPolicy + elif args.alg == "SAC": + if multiagent: + from hbaselines.multi_fcnet.sac import MultiFeedForwardPolicy + policy = MultiFeedForwardPolicy + else: + from hbaselines.fcnet.sac import FeedForwardPolicy + policy = FeedForwardPolicy + else: + raise ValueError("Unknown algorithm: {}".format(args.alg)) + + # Get the hyperparameters. + hp = get_hyperparameters(args, policy) + + # Add the seed for logging purposes. + params_with_extra = hp.copy() + params_with_extra['seed'] = seed + params_with_extra['env_name'] = args.env_name + params_with_extra['policy_name'] = policy.__name__ + params_with_extra['algorithm'] = args.alg + params_with_extra['date/time'] = now + + # Add the hyperparameters to the folder. + with open(os.path.join(dir_name, 'hyperparameters.json'), 'w') as f: + json.dump(params_with_extra, f, sort_keys=True, indent=4) + + # Create the algorithm object. + alg = OffPolicyRLAlgorithm( + policy=policy, + env="flow:{}".format(env_name), + eval_env="flow:{}".format(env_name) if args.evaluate else None, + **hp + ) + + # Perform training. + alg.learn( + total_steps=args.total_steps, + log_dir=dir_name, + log_interval=args.log_interval, + eval_interval=args.eval_interval, + save_interval=args.save_interval, + initial_exploration_steps=args.initial_exploration_steps, + seed=seed, + ) + + +def train_stable_baselines(submodule, flags): + """Train policies using the PPO algorithm in stable-baselines.""" + from stable_baselines.common.vec_env import DummyVecEnv + from stable_baselines import PPO2 + + flow_params = submodule.flow_params + # Path to the saved files + exp_tag = flow_params['exp_tag'] + result_name = '{}/{}'.format(exp_tag, strftime("%Y-%m-%d-%H:%M:%S")) + + # Perform training. + print('Beginning training.') + model = run_model_stablebaseline( + flow_params, flags.num_cpus, flags.rollout_size, flags.num_steps) + + # Save the model to a desired folder and then delete it to demonstrate + # loading. + print('Saving the trained model!') + path = os.path.realpath(os.path.expanduser('~/baseline_results')) + ensure_dir(path) + save_path = os.path.join(path, result_name) + model.save(save_path) + + # dump the flow params + with open(os.path.join(path, result_name) + '.json', 'w') as outfile: + json.dump(flow_params, outfile, + cls=FlowParamsEncoder, sort_keys=True, indent=4) + + # Replay the result by loading the model + print('Loading the trained model and testing it out!') + model = PPO2.load(save_path) + flow_params = get_flow_params(os.path.join(path, result_name) + '.json') + flow_params['sim'].render = True + env = env_constructor(params=flow_params, version=0)() + # The algorithms require a vectorized environment to run + eval_env = DummyVecEnv([lambda: env]) + obs = eval_env.reset() + reward = 0 + for _ in range(flow_params['env'].horizon): + action, _states = model.predict(obs) + obs, rewards, dones, info = eval_env.step(action) + reward += rewards + print('the final reward is {}'.format(reward)) + + +def main(args): + """Perform the training operations.""" + # Parse script-level arguments (not including package arguments). + flags = parse_args(args) + + # Import relevant information from the exp_config script. + module = __import__( + "exp_configs.rl.singleagent", fromlist=[flags.exp_config]) + module_ma = __import__( + "exp_configs.rl.multiagent", fromlist=[flags.exp_config]) + + # Import the sub-module containing the specified exp_config and determine + # whether the environment is single agent or multi-agent. + if hasattr(module, flags.exp_config): + submodule = getattr(module, flags.exp_config) + multiagent = False + elif hasattr(module_ma, flags.exp_config): + submodule = getattr(module_ma, flags.exp_config) + assert flags.rl_trainer.lower() in ["rllib", "h-baselines"], \ + "Currently, multiagent experiments are only supported through "\ + "RLlib. Try running this experiment using RLlib: " \ + "'python train.py EXP_CONFIG'" + multiagent = True + else: + raise ValueError("Unable to find experiment config.") + + # Perform the training operation. + if flags.rl_trainer.lower() == "rllib": + train_rllib(submodule, flags) + elif flags.rl_trainer.lower() == "stable-baselines": + train_stable_baselines(submodule, flags) + elif flags.rl_trainer.lower() == "h-baselines": + train_h_baselines(flags.exp_config, args, multiagent) + else: + raise ValueError("rl_trainer should be either 'rllib', 'h-baselines', " + "or 'stable-baselines'.") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/flow/benchmarks/README.md b/flow/benchmarks/README.md index 963ad5b70..bbcba9414 100644 --- a/flow/benchmarks/README.md +++ b/flow/benchmarks/README.md @@ -38,12 +38,12 @@ inflow = 300 veh/hour/lane S=(915,), A=(25,), T=400. this problem is to learn to avoid the *capacity drop* that is characteristic to bottleneck structures in transportation networks, and maximize the total outflow in a mixed-autonomy setting. -- `flow.benchmarks.bottleneck0` 4 lanes, inflow = 1900 veh/hour, 10% CAV +- `flow.benchmarks.bottleneck0` 4 lanes, inflow = 2500 veh/hour, 10% CAV penetration, no vehicles are allowed to lane change, S=(141,), A=(20,), T=1000. -- `flow.benchmarks.bottleneck1` 4 lanes, inflow = 1900 veh/hour, 10% CAV +- `flow.benchmarks.bottleneck1` 4 lanes, inflow = 2500 veh/hour, 10% CAV penetration, the human drivers follow the standard lane changing model in the simulator, S=(141,), A=(20,), T=1000. -- `flow.benchmarks.bottleneck2` 8 lanes, inflow = 3800 veh/hour, 10% CAV +- `flow.benchmarks.bottleneck2` 8 lanes, inflow = 5000 veh/hour, 10% CAV penetration, no vehicles are allowed to lane change, S=(281,), A=(40,), T=1000. ## Training on Custom Algorithms diff --git a/flow/benchmarks/baselines/bottleneck0.py b/flow/benchmarks/baselines/bottleneck0.py index 3df9f8350..1c29424fc 100644 --- a/flow/benchmarks/baselines/bottleneck0.py +++ b/flow/benchmarks/baselines/bottleneck0.py @@ -5,12 +5,10 @@ import numpy as np from flow.core.experiment import Experiment -from flow.core.params import InitialConfig from flow.core.params import InFlows from flow.core.params import SumoLaneChangeParams from flow.core.params import SumoCarFollowingParams from flow.core.params import VehicleParams -from flow.core.params import TrafficLightParams from flow.controllers import ContinuousRouter from flow.benchmarks.bottleneck0 import flow_params from flow.benchmarks.bottleneck0 import SCALING @@ -32,12 +30,9 @@ def bottleneck0_baseline(num_runs, render=True): flow.core.experiment.Experiment class needed to run simulations """ - exp_tag = flow_params['exp_tag'] sim_params = flow_params['sim'] env_params = flow_params['env'] net_params = flow_params['net'] - initial_config = flow_params.get('initial', InitialConfig()) - traffic_lights = flow_params.get('tls', TrafficLightParams()) # we want no autonomous vehicles in the simulation vehicles = VehicleParams() @@ -65,27 +60,10 @@ class needed to run simulations # set the evaluation flag to True env_params.evaluate = True - # import the network class - network_class = flow_params['network'] + flow_params['env'].horizon = env_params.horizon + exp = Experiment(flow_params) - # create the network object - network = network_class( - name=exp_tag, - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config, - traffic_lights=traffic_lights - ) - - # import the environment class - env_class = flow_params['env_name'] - - # create the environment object - env = env_class(env_params, sim_params, network) - - exp = Experiment(env) - - results = exp.run(num_runs, env_params.horizon) + results = exp.run(num_runs) return np.mean(results['returns']), np.std(results['returns']) diff --git a/flow/benchmarks/baselines/bottleneck1.py b/flow/benchmarks/baselines/bottleneck1.py index 2ad0e6428..a4aaa6dc3 100644 --- a/flow/benchmarks/baselines/bottleneck1.py +++ b/flow/benchmarks/baselines/bottleneck1.py @@ -5,12 +5,10 @@ import numpy as np from flow.core.experiment import Experiment -from flow.core.params import InitialConfig from flow.core.params import InFlows from flow.core.params import SumoLaneChangeParams from flow.core.params import SumoCarFollowingParams from flow.core.params import VehicleParams -from flow.core.params import TrafficLightParams from flow.controllers import ContinuousRouter from flow.benchmarks.bottleneck1 import flow_params from flow.benchmarks.bottleneck1 import SCALING @@ -32,12 +30,9 @@ def bottleneck1_baseline(num_runs, render=True): flow.core.experiment.Experiment class needed to run simulations """ - exp_tag = flow_params['exp_tag'] sim_params = flow_params['sim'] env_params = flow_params['env'] net_params = flow_params['net'] - initial_config = flow_params.get('initial', InitialConfig()) - traffic_lights = flow_params.get('tls', TrafficLightParams()) # we want no autonomous vehicles in the simulation vehicles = VehicleParams() @@ -65,27 +60,10 @@ class needed to run simulations # set the evaluation flag to True env_params.evaluate = True - # import the network class - network_class = flow_params['network'] + flow_params['env'].horizon = env_params.horizon + exp = Experiment(flow_params) - # create the network object - network = network_class( - name=exp_tag, - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config, - traffic_lights=traffic_lights - ) - - # import the environment class - env_class = flow_params['env_name'] - - # create the environment object - env = env_class(env_params, sim_params, network) - - exp = Experiment(env) - - results = exp.run(num_runs, env_params.horizon) + results = exp.run(num_runs) return np.mean(results['returns']), np.std(results['returns']) diff --git a/flow/benchmarks/baselines/bottleneck2.py b/flow/benchmarks/baselines/bottleneck2.py index c6343a61d..489d256b5 100644 --- a/flow/benchmarks/baselines/bottleneck2.py +++ b/flow/benchmarks/baselines/bottleneck2.py @@ -5,12 +5,10 @@ import numpy as np from flow.core.experiment import Experiment -from flow.core.params import InitialConfig from flow.core.params import InFlows from flow.core.params import SumoLaneChangeParams from flow.core.params import SumoCarFollowingParams from flow.core.params import VehicleParams -from flow.core.params import TrafficLightParams from flow.controllers import ContinuousRouter from flow.benchmarks.bottleneck2 import flow_params from flow.benchmarks.bottleneck2 import SCALING @@ -32,12 +30,9 @@ def bottleneck2_baseline(num_runs, render=True): flow.core.experiment.Experiment class needed to run simulations """ - exp_tag = flow_params['exp_tag'] sim_params = flow_params['sim'] env_params = flow_params['env'] net_params = flow_params['net'] - initial_config = flow_params.get('initial', InitialConfig()) - traffic_lights = flow_params.get('tls', TrafficLightParams()) # we want no autonomous vehicles in the simulation vehicles = VehicleParams() @@ -65,27 +60,10 @@ class needed to run simulations # set the evaluation flag to True env_params.evaluate = True - # import the network class - network_class = flow_params['network'] + flow_params['env'].horizon = env_params.horizon + exp = Experiment(flow_params) - # create the network object - network = network_class( - name=exp_tag, - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config, - traffic_lights=traffic_lights - ) - - # import the environment class - env_class = flow_params['env_name'] - - # create the environment object - env = env_class(env_params, sim_params, network) - - exp = Experiment(env) - - results = exp.run(num_runs, env_params.horizon) + results = exp.run(num_runs) return np.mean(results['returns']), np.std(results['returns']) diff --git a/flow/benchmarks/baselines/figureeight012.py b/flow/benchmarks/baselines/figureeight012.py index 19218c168..d18a968fa 100644 --- a/flow/benchmarks/baselines/figureeight012.py +++ b/flow/benchmarks/baselines/figureeight012.py @@ -5,10 +5,8 @@ import numpy as np from flow.core.experiment import Experiment -from flow.core.params import InitialConfig from flow.core.params import SumoCarFollowingParams from flow.core.params import VehicleParams -from flow.core.params import TrafficLightParams from flow.controllers import IDMController from flow.controllers import ContinuousRouter from flow.benchmarks.figureeight0 import flow_params @@ -30,12 +28,8 @@ def figure_eight_baseline(num_runs, render=True): Experiment class needed to run simulations """ - exp_tag = flow_params['exp_tag'] sim_params = flow_params['sim'] env_params = flow_params['env'] - net_params = flow_params['net'] - initial_config = flow_params.get('initial', InitialConfig()) - traffic_lights = flow_params.get('tls', TrafficLightParams()) # modify the rendering to match what is requested sim_params.render = render @@ -53,28 +47,11 @@ class needed to run simulations ), num_vehicles=14) - # import the network class - network_class = flow_params['network'] + flow_params['env'].horizon = env_params.horizon + exp = Experiment(flow_params) - # create the network object - network = network_class( - name=exp_tag, - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config, - traffic_lights=traffic_lights - ) - - # import the environment class - env_class = flow_params['env_name'] - - # create the environment object - env = env_class(env_params, sim_params, network) - - exp = Experiment(env) - - results = exp.run(num_runs, env_params.horizon) - avg_speed = np.mean(results['mean_returns']) + results = exp.run(num_runs) + avg_speed = np.mean(results['returns']) return avg_speed diff --git a/flow/benchmarks/baselines/grid0.py b/flow/benchmarks/baselines/grid0.py index eb27de4c6..05ca393ab 100644 --- a/flow/benchmarks/baselines/grid0.py +++ b/flow/benchmarks/baselines/grid0.py @@ -5,7 +5,6 @@ import numpy as np from flow.core.experiment import Experiment -from flow.core.params import InitialConfig from flow.core.params import TrafficLightParams from flow.benchmarks.grid0 import flow_params from flow.benchmarks.grid0 import N_ROWS @@ -28,12 +27,8 @@ def grid0_baseline(num_runs, render=True): flow.core.experiment.Experiment class needed to run simulations """ - exp_tag = flow_params['exp_tag'] sim_params = flow_params['sim'] - vehicles = flow_params['veh'] env_params = flow_params['env'] - net_params = flow_params['net'] - initial_config = flow_params.get('initial', InitialConfig()) # define the traffic light logic tl_logic = TrafficLightParams(baseline=False) @@ -57,27 +52,10 @@ class needed to run simulations # set the evaluation flag to True env_params.evaluate = True - # import the network class - network_class = flow_params['network'] + flow_params['env'].horizon = env_params.horizon + exp = Experiment(flow_params) - # create the network object - network = network_class( - name=exp_tag, - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config, - traffic_lights=tl_logic - ) - - # import the environment class - env_class = flow_params['env_name'] - - # create the environment object - env = env_class(env_params, sim_params, network) - - exp = Experiment(env) - - results = exp.run(num_runs, env_params.horizon) + results = exp.run(num_runs) total_delay = np.mean(results['returns']) return total_delay diff --git a/flow/benchmarks/baselines/grid1.py b/flow/benchmarks/baselines/grid1.py index 049a6995a..76aea4c7f 100644 --- a/flow/benchmarks/baselines/grid1.py +++ b/flow/benchmarks/baselines/grid1.py @@ -5,7 +5,6 @@ import numpy as np from flow.core.experiment import Experiment -from flow.core.params import InitialConfig from flow.core.params import TrafficLightParams from flow.benchmarks.grid1 import flow_params from flow.benchmarks.grid1 import N_ROWS @@ -28,12 +27,8 @@ def grid1_baseline(num_runs, render=True): flow.core.experiment.Experiment class needed to run simulations """ - exp_tag = flow_params['exp_tag'] sim_params = flow_params['sim'] - vehicles = flow_params['veh'] env_params = flow_params['env'] - net_params = flow_params['net'] - initial_config = flow_params.get('initial', InitialConfig()) # define the traffic light logic tl_logic = TrafficLightParams(baseline=False) @@ -55,27 +50,10 @@ class needed to run simulations # set the evaluation flag to True env_params.evaluate = True - # import the network class - network_class = flow_params['network'] + flow_params['env'].horizon = env_params.horizon + exp = Experiment(flow_params) - # create the network object - network = network_class( - name=exp_tag, - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config, - traffic_lights=tl_logic - ) - - # import the environment class - env_class = flow_params['env_name'] - - # create the environment object - env = env_class(env_params, sim_params, network) - - exp = Experiment(env) - - results = exp.run(num_runs, env_params.horizon) + results = exp.run(num_runs) total_delay = np.mean(results['returns']) return total_delay diff --git a/flow/benchmarks/baselines/merge012.py b/flow/benchmarks/baselines/merge012.py index 5bd4a3a39..7851dc31f 100644 --- a/flow/benchmarks/baselines/merge012.py +++ b/flow/benchmarks/baselines/merge012.py @@ -5,8 +5,6 @@ import numpy as np from flow.core.experiment import Experiment -from flow.core.params import InitialConfig -from flow.core.params import TrafficLightParams from flow.benchmarks.merge0 import flow_params @@ -26,13 +24,8 @@ def merge_baseline(num_runs, render=True): flow.core.experiment.Experiment class needed to run simulations """ - exp_tag = flow_params['exp_tag'] sim_params = flow_params['sim'] - vehicles = flow_params['veh'] env_params = flow_params['env'] - net_params = flow_params['net'] - initial_config = flow_params.get('initial', InitialConfig()) - traffic_lights = flow_params.get('tls', TrafficLightParams()) # modify the rendering to match what is requested sim_params.render = render @@ -40,28 +33,11 @@ class needed to run simulations # set the evaluation flag to True env_params.evaluate = True - # import the network class - network_class = flow_params['network'] + flow_params['env'].horizon = env_params.horizon + exp = Experiment(flow_params) - # create the network object - network = network_class( - name=exp_tag, - vehicles=vehicles, - net_params=net_params, - initial_config=initial_config, - traffic_lights=traffic_lights - ) - - # import the environment class - env_class = flow_params['env_name'] - - # create the environment object - env = env_class(env_params, sim_params, network) - - exp = Experiment(env) - - results = exp.run(num_runs, env_params.horizon) - avg_speed = np.mean(results['mean_returns']) + results = exp.run(num_runs) + avg_speed = np.mean(results['returns']) return avg_speed diff --git a/flow/benchmarks/bottleneck0.py b/flow/benchmarks/bottleneck0.py index b0e86844c..b07947ad7 100644 --- a/flow/benchmarks/bottleneck0.py +++ b/flow/benchmarks/bottleneck0.py @@ -66,7 +66,7 @@ } # flow rate -flow_rate = 2000 * SCALING +flow_rate = 2500 * SCALING # percentage of flow coming out of each lane inflow = InFlows() diff --git a/flow/benchmarks/bottleneck1.py b/flow/benchmarks/bottleneck1.py index 26ae6527a..9c8d9c192 100644 --- a/flow/benchmarks/bottleneck1.py +++ b/flow/benchmarks/bottleneck1.py @@ -66,7 +66,7 @@ } # flow rate -flow_rate = 2000 * SCALING +flow_rate = 2500 * SCALING # percentage of flow coming out of each lane inflow = InFlows() diff --git a/flow/benchmarks/bottleneck2.py b/flow/benchmarks/bottleneck2.py index 5052b3b88..4651d448b 100644 --- a/flow/benchmarks/bottleneck2.py +++ b/flow/benchmarks/bottleneck2.py @@ -66,7 +66,7 @@ } # flow rate -flow_rate = 2000 * SCALING +flow_rate = 2500 * SCALING # percentage of flow coming out of each lane inflow = InFlows() diff --git a/flow/benchmarks/grid0.py b/flow/benchmarks/grid0.py index 1655c3b3c..5c4ee5349 100644 --- a/flow/benchmarks/grid0.py +++ b/flow/benchmarks/grid0.py @@ -4,7 +4,7 @@ - **Observation Dimension**: (339, ) - **Horizon**: 400 steps """ -from flow.envs import TrafficLightGridPOEnv +from flow.envs import TrafficLightGridBenchmarkEnv from flow.networks import TrafficLightGridNetwork from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams, \ InFlows, SumoCarFollowingParams @@ -68,7 +68,7 @@ exp_tag="grid_0", # name of the flow environment the experiment is running on - env_name=TrafficLightGridPOEnv, + env_name=TrafficLightGridBenchmarkEnv, # name of the network class the experiment is running on network=TrafficLightGridNetwork, diff --git a/flow/benchmarks/grid1.py b/flow/benchmarks/grid1.py index ec2a27454..83055adfd 100644 --- a/flow/benchmarks/grid1.py +++ b/flow/benchmarks/grid1.py @@ -4,7 +4,7 @@ - **Observation Dimension**: (915, ) - **Horizon**: 400 steps """ -from flow.envs import TrafficLightGridPOEnv +from flow.envs import TrafficLightGridBenchmarkEnv from flow.networks import TrafficLightGridNetwork from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams, \ InFlows, SumoCarFollowingParams @@ -68,7 +68,7 @@ exp_tag="grid_1", # name of the flow environment the experiment is running on - env_name=TrafficLightGridPOEnv, + env_name=TrafficLightGridBenchmarkEnv, # name of the network class the experiment is running on network=TrafficLightGridNetwork, diff --git a/flow/benchmarks/rllib/ars_runner.py b/flow/benchmarks/rllib/ars_runner.py index f765611dd..c5bcf8566 100644 --- a/flow/benchmarks/rllib/ars_runner.py +++ b/flow/benchmarks/rllib/ars_runner.py @@ -22,7 +22,7 @@ EXAMPLE_USAGE = """ example usage: - python ars_runner.py grid0 + python ars_runner.py --benchmark_name=grid0 Here the arguments are: benchmark_name - name of the benchmark to run num_rollouts - number of rollouts to train across diff --git a/flow/benchmarks/rllib/es_runner.py b/flow/benchmarks/rllib/es_runner.py index e71c01bef..2dd566679 100644 --- a/flow/benchmarks/rllib/es_runner.py +++ b/flow/benchmarks/rllib/es_runner.py @@ -19,7 +19,7 @@ EXAMPLE_USAGE = """ example usage: - python es_runner.py grid0 + python es_runner.py --benchmark_name=grid0 Here the arguments are: benchmark_name - name of the benchmark to run num_rollouts - number of rollouts to train across diff --git a/flow/benchmarks/rllib/ppo_runner.py b/flow/benchmarks/rllib/ppo_runner.py index 401955470..f53d86c8c 100644 --- a/flow/benchmarks/rllib/ppo_runner.py +++ b/flow/benchmarks/rllib/ppo_runner.py @@ -21,7 +21,7 @@ EXAMPLE_USAGE = """ example usage: - python ppo_runner.py grid0 + python ppo_runner.py --benchmark_name=grid0 Here the arguments are: benchmark_name - name of the benchmark to run num_rollouts - number of rollouts to train across diff --git a/flow/config.py b/flow/config.py index 7130f6d8a..94c99f473 100644 --- a/flow/config.py +++ b/flow/config.py @@ -21,6 +21,7 @@ # path to the Aimsun_Next main directory (required for Aimsun simulations) AIMSUN_NEXT_PATH = os.environ.get("AIMSUN_NEXT_PATH", None) + # path to the aimsun_flow environment's main directory (required for Aimsun # simulations) AIMSUN_SITEPACKAGES = os.environ.get("AIMSUN_SITEPACKAGES", None) diff --git a/flow/controllers/__init__.py b/flow/controllers/__init__.py index e5899b702..a61d16980 100755 --- a/flow/controllers/__init__.py +++ b/flow/controllers/__init__.py @@ -14,9 +14,10 @@ from flow.controllers.base_controller import BaseController from flow.controllers.car_following_models import CFMController, \ BCMController, OVMController, LinearOVM, IDMController, \ - SimCarFollowingController, LACController + SimCarFollowingController, LACController, GippsController, \ + BandoFTLController from flow.controllers.velocity_controllers import FollowerStopper, \ - PISaturation + PISaturation, NonLocalFollowerStopper # lane change controllers from flow.controllers.base_lane_changing_controller import \ @@ -27,12 +28,14 @@ # routing controllers from flow.controllers.base_routing_controller import BaseRouter from flow.controllers.routing_controllers import ContinuousRouter, \ - GridRouter, BayBridgeRouter + GridRouter, BayBridgeRouter, I210Router __all__ = [ "RLController", "BaseController", "BaseLaneChangeController", "BaseRouter", "CFMController", "BCMController", "OVMController", "LinearOVM", "IDMController", "SimCarFollowingController", "FollowerStopper", "PISaturation", "StaticLaneChanger", "SimLaneChangeController", - "ContinuousRouter", "GridRouter", "BayBridgeRouter", "LACController" + "ContinuousRouter", "GridRouter", "BayBridgeRouter", "LACController", + "GippsController", "NonLocalFollowerStopper", "BandoFTLController", + "I210Router" ] diff --git a/flow/controllers/base_controller.py b/flow/controllers/base_controller.py index 41780826b..3c9985360 100755 --- a/flow/controllers/base_controller.py +++ b/flow/controllers/base_controller.py @@ -1,9 +1,10 @@ """Contains the base acceleration controller class.""" +from abc import ABCMeta, abstractmethod import numpy as np -class BaseController: +class BaseController(metaclass=ABCMeta): """Base class for flow-controlled acceleration behavior. Instantiates a controller and forces the user to pass a @@ -33,8 +34,12 @@ class BaseController: specified to in this model are as desired. delay : int delay in applying the action (time) - fail_safe : str - Should be either "instantaneous" or "safe_velocity" + fail_safe : list of str or str + List of failsafes which can be "instantaneous", "safe_velocity", + "feasible_accel", or "obey_speed_limit". The order of applying the + falsafes will be based on the order in the list. + display_warnings : bool + Flag for toggling on/off printing failsafe warnings to screen. noise : double variance of the gaussian from which to sample a noisy acceleration """ @@ -44,6 +49,7 @@ def __init__(self, car_following_params, delay=0, fail_safe=None, + display_warnings=True, noise=0): """Instantiate the base class for acceleration behavior.""" self.veh_id = veh_id @@ -55,7 +61,29 @@ def __init__(self, self.delay = delay # longitudinal failsafe used by the vehicle - self.fail_safe = fail_safe + if isinstance(fail_safe, str): + failsafe_list = [fail_safe] + elif isinstance(fail_safe, list) or fail_safe is None: + failsafe_list = fail_safe + else: + failsafe_list = None + raise ValueError("fail_safe should be string or list of strings. Setting fail_safe to None\n") + + failsafe_map = { + 'instantaneous': self.get_safe_action_instantaneous, + 'safe_velocity': self.get_safe_velocity_action, + 'feasible_accel': lambda _, accel: self.get_feasible_action(accel), + 'obey_speed_limit': self.get_obey_speed_limit_action + } + self.failsafes = [] + if failsafe_list: + for check in failsafe_list: + if check in failsafe_map: + self.failsafes.append(failsafe_map.get(check)) + else: + raise ValueError('Skipping {}, as it is not a valid failsafe.'.format(check)) + + self.display_warnings = display_warnings self.max_accel = car_following_params.controller_params['accel'] # max deaccel should always be a positive @@ -63,9 +91,10 @@ def __init__(self, self.car_following_params = car_following_params + @abstractmethod def get_accel(self, env): """Return the acceleration of the controller.""" - raise NotImplementedError + pass def get_action(self, env): """Convert the get_accel() acceleration into an action. @@ -75,8 +104,8 @@ def get_action(self, env): time step. This method also augments the controller with the desired level of - stochastic noise, and utlizes the "instantaneous" or "safe_velocity" - failsafes if requested. + stochastic noise, and utlizes the "instantaneous", "safe_velocity", + "feasible_accel", and/or "obey_speed_limit" failsafes if requested. Parameters ---------- @@ -88,6 +117,12 @@ def get_action(self, env): float the modified form of the acceleration """ + # clear the current stored accels of this vehicle to None + env.k.vehicle.update_accel(self.veh_id, None, noise=False, failsafe=False) + env.k.vehicle.update_accel(self.veh_id, None, noise=False, failsafe=True) + env.k.vehicle.update_accel(self.veh_id, None, noise=True, failsafe=False) + env.k.vehicle.update_accel(self.veh_id, None, noise=True, failsafe=True) + # this is to avoid abrupt decelerations when a vehicle has just entered # a network and it's data is still not subscribed if len(env.k.vehicle.get_edge(self.veh_id)) == 0: @@ -105,16 +140,26 @@ def get_action(self, env): if accel is None: return None + # store the acceleration without noise to each vehicle + # run fail safe if requested + env.k.vehicle.update_accel(self.veh_id, accel, noise=False, failsafe=False) + accel_no_noise_with_failsafe = accel + + for failsafe in self.failsafes: + accel_no_noise_with_failsafe = failsafe(env, accel_no_noise_with_failsafe) + + env.k.vehicle.update_accel(self.veh_id, accel_no_noise_with_failsafe, noise=False, failsafe=True) + # add noise to the accelerations, if requested if self.accel_noise > 0: - accel += np.random.normal(0, self.accel_noise) + accel += np.sqrt(env.sim_step) * np.random.normal(0, self.accel_noise) + env.k.vehicle.update_accel(self.veh_id, accel, noise=True, failsafe=False) - # run the failsafes, if requested - if self.fail_safe == 'instantaneous': - accel = self.get_safe_action_instantaneous(env, accel) - elif self.fail_safe == 'safe_velocity': - accel = self.get_safe_velocity_action(env, accel) + # run the fail-safes, if requested + for failsafe in self.failsafes: + accel = failsafe(env, accel) + env.k.vehicle.update_accel(self.veh_id, accel, noise=True, failsafe=True) return accel def get_safe_action_instantaneous(self, env, action): @@ -160,6 +205,13 @@ def get_safe_action_instantaneous(self, env, action): # if the vehicle will crash into the vehicle ahead of it in the # next time step (assuming the vehicle ahead of it is not # moving), then stop immediately + if self.display_warnings: + print( + "=====================================\n" + "Vehicle {} is about to crash. Instantaneous acceleration " + "clipping applied.\n" + "=====================================".format(self.veh_id)) + return -this_vel / sim_step else: # if the vehicle is not in danger of crashing, continue with @@ -221,8 +273,8 @@ def safe_velocity(self, env): Returns ------- float - maximum safe velocity given a maximum deceleration and delay in - performing the breaking action + maximum safe velocity given a maximum deceleration, delay in + performing the breaking action, and speed limit """ lead_id = env.k.vehicle.get_leader(self.veh_id) lead_vel = env.k.vehicle.get_speed(lead_id) @@ -233,4 +285,97 @@ def safe_velocity(self, env): v_safe = 2 * h / env.sim_step + dv - this_vel * (2 * self.delay) + # check for speed limit FIXME: this is not called + # this_edge = env.k.vehicle.get_edge(self.veh_id) + # edge_speed_limit = env.k.network.speed_limit(this_edge) + + if this_vel > v_safe: + if self.display_warnings: + print( + "=====================================\n" + "Speed of vehicle {} is greater than safe speed. Safe velocity " + "clipping applied.\n" + "=====================================".format(self.veh_id)) + return v_safe + + def get_obey_speed_limit_action(self, env, action): + """Perform the "obey_speed_limit" failsafe action. + + Checks if the computed acceleration would put us above edge speed limit. + If it would, output the acceleration that would put at the speed limit + velocity. + + Parameters + ---------- + env : flow.envs.Env + current environment, which contains information of the state of the + network at the current time step + action : float + requested acceleration action + + Returns + ------- + float + the requested action clipped by the speed limit + """ + # check for speed limit + this_edge = env.k.vehicle.get_edge(self.veh_id) + edge_speed_limit = env.k.network.speed_limit(this_edge) + + this_vel = env.k.vehicle.get_speed(self.veh_id) + sim_step = env.sim_step + + if this_vel + action * sim_step > edge_speed_limit: + if edge_speed_limit > 0: + if self.display_warnings: + print( + "=====================================\n" + "Speed of vehicle {} is greater than speed limit. Obey " + "speed limit clipping applied.\n" + "=====================================".format(self.veh_id)) + return (edge_speed_limit - this_vel) / sim_step + else: + return -this_vel / sim_step + else: + return action + + def get_feasible_action(self, action): + """Perform the "feasible_accel" failsafe action. + + Checks if the computed acceleration would put us above maximum + acceleration or deceleration. If it would, output the acceleration + equal to maximum acceleration or deceleration. + + Parameters + ---------- + action : float + requested acceleration action + + Returns + ------- + float + the requested action clipped by the feasible acceleration or + deceleration. + """ + if action > self.max_accel: + action = self.max_accel + + if self.display_warnings: + print( + "=====================================\n" + "Acceleration of vehicle {} is greater than the max " + "acceleration. Feasible acceleration clipping applied.\n" + "=====================================".format(self.veh_id)) + + if action < -self.max_deaccel: + action = -self.max_deaccel + + if self.display_warnings: + print( + "=====================================\n" + "Deceleration of vehicle {} is greater than the max " + "deceleration. Feasible acceleration clipping applied.\n" + "=====================================".format(self.veh_id)) + + return action diff --git a/flow/controllers/base_lane_changing_controller.py b/flow/controllers/base_lane_changing_controller.py index af009f992..eb2b566f5 100755 --- a/flow/controllers/base_lane_changing_controller.py +++ b/flow/controllers/base_lane_changing_controller.py @@ -1,7 +1,9 @@ """Contains the base lane change controller class.""" +from abc import ABCMeta, abstractmethod -class BaseLaneChangeController: + +class BaseLaneChangeController(metaclass=ABCMeta): """Base class for lane-changing controllers. Instantiates a controller and forces the user to pass a @@ -36,6 +38,7 @@ def __init__(self, veh_id, lane_change_params=None): self.veh_id = veh_id self.lane_change_params = lane_change_params + @abstractmethod def get_lane_change_action(self, env): """Specify the lane change action to be performed. @@ -55,7 +58,7 @@ def get_lane_change_action(self, env): float or int requested lane change action """ - raise NotImplementedError + pass def get_action(self, env): """Return the action of the lane change controller. diff --git a/flow/controllers/base_routing_controller.py b/flow/controllers/base_routing_controller.py index b001dc62e..17048ce7d 100755 --- a/flow/controllers/base_routing_controller.py +++ b/flow/controllers/base_routing_controller.py @@ -1,7 +1,9 @@ """Contains the base routing controller class.""" +from abc import ABCMeta, abstractmethod -class BaseRouter: + +class BaseRouter(metaclass=ABCMeta): """Base class for routing controllers. These controllers are used to dynamically change the routes of vehicles @@ -30,6 +32,7 @@ def __init__(self, veh_id, router_params): self.veh_id = veh_id self.router_params = router_params + @abstractmethod def choose_route(self, env): """Return the routing method implemented by the controller. @@ -45,4 +48,4 @@ def choose_route(self, env): is returned, the vehicle performs no routing action in the current time step. """ - raise NotImplementedError + pass diff --git a/flow/controllers/car_following_models.py b/flow/controllers/car_following_models.py index 50b0ceac6..f5b7399bc 100755 --- a/flow/controllers/car_following_models.py +++ b/flow/controllers/car_following_models.py @@ -56,7 +56,8 @@ def __init__(self, v_des=8, time_delay=0.0, noise=0, - fail_safe=None): + fail_safe=None, + display_warnings=True): """Instantiate a CFM controller.""" BaseController.__init__( self, @@ -64,7 +65,9 @@ def __init__(self, car_following_params, delay=time_delay, fail_safe=fail_safe, - noise=noise) + noise=noise, + display_warnings=display_warnings, + ) self.veh_id = veh_id self.k_d = k_d @@ -132,7 +135,8 @@ def __init__(self, v_des=8, time_delay=0.0, noise=0, - fail_safe=None): + fail_safe=None, + display_warnings=True): """Instantiate a Bilateral car-following model controller.""" BaseController.__init__( self, @@ -140,7 +144,9 @@ def __init__(self, car_following_params, delay=time_delay, fail_safe=fail_safe, - noise=noise) + noise=noise, + display_warnings=display_warnings, + ) self.veh_id = veh_id self.k_d = k_d @@ -212,7 +218,8 @@ def __init__(self, a=0, time_delay=0.0, noise=0, - fail_safe=None): + fail_safe=None, + display_warnings=True): """Instantiate a Linear Adaptive Cruise controller.""" BaseController.__init__( self, @@ -220,7 +227,9 @@ def __init__(self, car_following_params, delay=time_delay, fail_safe=fail_safe, - noise=noise) + noise=noise, + display_warnings=display_warnings, + ) self.veh_id = veh_id self.k_1 = k_1 @@ -289,7 +298,8 @@ def __init__(self, v_max=30, time_delay=0, noise=0, - fail_safe=None): + fail_safe=None, + display_warnings=True): """Instantiate an Optimal Vehicle Model controller.""" BaseController.__init__( self, @@ -297,7 +307,9 @@ def __init__(self, car_following_params, delay=time_delay, fail_safe=fail_safe, - noise=noise) + noise=noise, + display_warnings=display_warnings, + ) self.veh_id = veh_id self.v_max = v_max self.alpha = alpha @@ -364,7 +376,8 @@ def __init__(self, h_st=5, time_delay=0.0, noise=0, - fail_safe=None): + fail_safe=None, + display_warnings=True): """Instantiate a Linear OVM controller.""" BaseController.__init__( self, @@ -372,7 +385,9 @@ def __init__(self, car_following_params, delay=time_delay, fail_safe=fail_safe, - noise=noise) + noise=noise, + display_warnings=display_warnings, + ) self.veh_id = veh_id # 4.8*1.85 for case I, 3.8*1.85 for case II, per Nakayama self.v_max = v_max @@ -427,8 +442,6 @@ class IDMController(BaseController): acceleration exponent (default: 4) s0 : float linear jam distance, in m (default: 2) - dt : float - timestep, in s (default: 0.1) noise : float std dev of normal perturbation to the acceleration (default: 0) fail_safe : str @@ -445,9 +458,9 @@ def __init__(self, delta=4, s0=2, time_delay=0.0, - dt=0.1, noise=0, fail_safe=None, + display_warnings=True, car_following_params=None): """Instantiate an IDM controller.""" BaseController.__init__( @@ -456,14 +469,15 @@ def __init__(self, car_following_params, delay=time_delay, fail_safe=fail_safe, - noise=noise) + noise=noise, + display_warnings=display_warnings, + ) self.v0 = v0 self.T = T self.a = a self.b = b self.delta = delta self.s0 = s0 - self.dt = dt def get_accel(self, env): """See parent class.""" @@ -499,3 +513,175 @@ class SimCarFollowingController(BaseController): def get_accel(self, env): """See parent class.""" return None + + +class GippsController(BaseController): + """Gipps' Model controller. + + For more information on this controller, see: + Traffic Flow Dynamics written by M.Treiber and A.Kesting + By courtesy of Springer publisher, http://www.springer.com + + http://www.traffic-flow-dynamics.org/res/SampleChapter11.pdf + + Usage + ----- + See BaseController for usage example. + + Attributes + ---------- + veh_id : str + Vehicle ID for SUMO identification + car_following_params : flow.core.param.SumoCarFollowingParams + see parent class + v0 : float + desirable velocity, in m/s (default: 30) + acc : float + max acceleration, in m/s2 (default: 1.5) + b : float + comfortable deceleration, in m/s2 (default: -1) + b_l : float + comfortable deceleration for leading vehicle , in m/s2 (default: -1) + s0 : float + linear jam distance for saftey, in m (default: 2) + tau : float + reaction time in s (default: 1) + noise : float + std dev of normal perturbation to the acceleration (default: 0) + fail_safe : str + type of flow-imposed failsafe the vehicle should posses, defaults + to no failsafe (None) + """ + + def __init__(self, + veh_id, + car_following_params=None, + v0=30, + acc=1.5, + b=-1, + b_l=-1, + s0=2, + tau=1, + delay=0, + noise=0, + fail_safe=None, + display_warnings=True): + """Instantiate a Gipps' controller.""" + BaseController.__init__( + self, + veh_id, + car_following_params, + delay=delay, + fail_safe=fail_safe, + noise=noise, + display_warnings=display_warnings, + ) + + self.v_desired = v0 + self.acc = acc + self.b = b + self.b_l = b_l + self.s0 = s0 + self.tau = tau + + def get_accel(self, env): + """See parent class.""" + v = env.k.vehicle.get_speed(self.veh_id) + h = env.k.vehicle.get_headway(self.veh_id) + v_l = env.k.vehicle.get_speed( + env.k.vehicle.get_leader(self.veh_id)) + + # get velocity dynamics + v_acc = v + (2.5 * self.acc * self.tau * ( + 1 - (v / self.v_desired)) * np.sqrt(0.025 + (v / self.v_desired))) + v_safe = (self.tau * self.b) + np.sqrt(((self.tau**2) * (self.b**2)) - ( + self.b * ((2 * (h-self.s0)) - (self.tau * v) - ((v_l**2) / self.b_l)))) + + v_next = min(v_acc, v_safe, self.v_desired) + + return (v_next-v)/env.sim_step + + +class BandoFTLController(BaseController): + """Bando follow-the-leader controller. + + Usage + ----- + See BaseController for usage example. + + Attributes + ---------- + veh_id : str + Vehicle ID for SUMO identification + car_following_params : flow.core.params.SumoCarFollowingParams + see parent class + alpha : float + gain on desired velocity to current velocity difference + (default: 0.6) + beta : float + gain on lead car velocity and self velocity difference + (default: 0.9) + h_st : float + headway for stopping (default: 5) + h_go : float + headway for full speed (default: 35) + v_max : float + max velocity (default: 30) + time_delay : float + time delay (default: 0.5) + noise : float + std dev of normal perturbation to the acceleration (default: 0) + fail_safe : str + type of flow-imposed failsafe the vehicle should posses, defaults + to no failsafe (None) + """ + + def __init__(self, + veh_id, + car_following_params, + alpha=.5, + beta=20, + h_st=2, + h_go=10, + v_max=32, + want_max_accel=False, + time_delay=0, + noise=0, + fail_safe=None, + display_warnings=True): + """Instantiate an Bando controller.""" + BaseController.__init__( + self, + veh_id, + car_following_params, + delay=time_delay, + fail_safe=fail_safe, + noise=noise, + display_warnings=display_warnings, + ) + self.veh_id = veh_id + self.v_max = v_max + self.alpha = alpha + self.beta = beta + self.h_st = h_st + self.h_go = h_go + self.want_max_accel = want_max_accel + + def get_accel(self, env): + """See parent class.""" + lead_id = env.k.vehicle.get_leader(self.veh_id) + if not lead_id: # no car ahead + if self.want_max_accel: + return self.max_accel + + v_l = env.k.vehicle.get_speed(lead_id) + v = env.k.vehicle.get_speed(self.veh_id) + s = env.k.vehicle.get_headway(self.veh_id) + return self.accel_func(v, v_l, s) + + def accel_func(self, v, v_l, s): + """Compute the acceleration function.""" + v_h = self.v_max * ((np.tanh(s/self.h_st-2)+np.tanh(2))/(1+np.tanh(2))) + s_dot = v_l - v + u = self.alpha * (v_h - v) + self.beta * s_dot/(s**2) + return u diff --git a/flow/controllers/rlcontroller.py b/flow/controllers/rlcontroller.py index 61f53f11a..973de8fc9 100755 --- a/flow/controllers/rlcontroller.py +++ b/flow/controllers/rlcontroller.py @@ -37,3 +37,7 @@ def __init__(self, veh_id, car_following_params): self, veh_id, car_following_params) + + def get_accel(self, env): + """Pass, as this is never called; required to override abstractmethod.""" + pass diff --git a/flow/controllers/routing_controllers.py b/flow/controllers/routing_controllers.py index f9b346fdd..02aa34cb4 100755 --- a/flow/controllers/routing_controllers.py +++ b/flow/controllers/routing_controllers.py @@ -1,5 +1,4 @@ """Contains a list of custom routing controllers.""" - import random import numpy as np @@ -125,3 +124,29 @@ def choose_route(self, env): new_route = super().choose_route(env) return new_route + + +class I210Router(ContinuousRouter): + """Assists in choosing routes in select cases for the I-210 sub-network. + + Extension to the Continuous Router. + + Usage + ----- + See base class for usage example. + """ + + def choose_route(self, env): + """See parent class.""" + edge = env.k.vehicle.get_edge(self.veh_id) + lane = env.k.vehicle.get_lane(self.veh_id) + + # vehicles on these edges in lanes 4 and 5 are not going to be able to + # make it out in time + if edge == "119257908#1-AddedOffRampEdge" and lane in [5, 4, 3]: + new_route = env.available_routes[ + "119257908#1-AddedOffRampEdge"][0][0] + else: + new_route = super().choose_route(env) + + return new_route diff --git a/flow/controllers/velocity_controllers.py b/flow/controllers/velocity_controllers.py index 8ce2710cc..2e4b7c22a 100644 --- a/flow/controllers/velocity_controllers.py +++ b/flow/controllers/velocity_controllers.py @@ -116,6 +116,54 @@ def get_accel(self, env): return (v_cmd - this_vel) / env.sim_step +class NonLocalFollowerStopper(FollowerStopper): + """Follower stopper that uses the average system speed to compute its acceleration.""" + + def get_accel(self, env): + """See parent class.""" + lead_id = env.k.vehicle.get_leader(self.veh_id) + this_vel = env.k.vehicle.get_speed(self.veh_id) + lead_vel = env.k.vehicle.get_speed(lead_id) + self.v_des = np.mean(env.k.vehicle.get_speed(env.k.vehicle.get_ids())) + + if self.v_des is None: + return None + + if lead_id is None: + v_cmd = self.v_des + else: + dx = env.k.vehicle.get_headway(self.veh_id) + dv_minus = min(lead_vel - this_vel, 0) + + dx_1 = self.dx_1_0 + 1 / (2 * self.d_1) * dv_minus ** 2 + dx_2 = self.dx_2_0 + 1 / (2 * self.d_2) * dv_minus ** 2 + dx_3 = self.dx_3_0 + 1 / (2 * self.d_3) * dv_minus ** 2 + v = min(max(lead_vel, 0), self.v_des) + # compute the desired velocity + if dx <= dx_1: + v_cmd = 0 + elif dx <= dx_2: + v_cmd = v * (dx - dx_1) / (dx_2 - dx_1) + elif dx <= dx_3: + v_cmd = v + (self.v_des - this_vel) * (dx - dx_2) \ + / (dx_3 - dx_2) + else: + v_cmd = self.v_des + + edge = env.k.vehicle.get_edge(self.veh_id) + + if edge == "": + return None + + if self.find_intersection_dist(env) <= 10 and \ + env.k.vehicle.get_edge(self.veh_id) in self.danger_edges or \ + env.k.vehicle.get_edge(self.veh_id)[0] == ":": + return None + else: + # compute the acceleration from the desired velocity + return (v_cmd - this_vel) / env.sim_step + + class PISaturation(BaseController): """Inspired by Dan Work's... work. diff --git a/flow/core/experiment.py b/flow/core/experiment.py index 2d53e9463..d97f96582 100755 --- a/flow/core/experiment.py +++ b/flow/core/experiment.py @@ -1,12 +1,9 @@ """Contains an experiment class for running simulations.""" - +from flow.utils.registry import make_create_env +from datetime import datetime import logging -import datetime -import numpy as np import time -import os - -from flow.core.util import emission_to_csv +import numpy as np class Experiment: @@ -18,15 +15,15 @@ class Experiment: the actions of RL agents in the network, type the following: >>> from flow.envs import Env - >>> env = Env(...) - >>> exp = Experiment(env) # for some env - >>> exp.run(num_runs=1, num_steps=1000) + >>> flow_params = dict(...) # see the examples in exp_config + >>> exp = Experiment(flow_params) # for some experiment configuration + >>> exp.run(num_runs=1) If you wish to specify the actions of RL agents in the network, this may be done as follows: >>> rl_actions = lambda state: 0 # replace with something appropriate - >>> exp.run(num_runs=1, num_steps=1000, rl_actions=rl_actions) + >>> exp.run(num_runs=1, rl_actions=rl_actions) Finally, if you would like to like to plot and visualize your results, this class can generate csv files from emission files produced by sumo. These @@ -37,12 +34,12 @@ class can generate csv files from emission files produced by sumo. These ``emission_path`` attribute in ``SimParams`` to some path. >>> from flow.core.params import SimParams - >>> sim_params = SimParams(emission_path="./data") + >>> flow_params['sim'] = SimParams(emission_path="./data") Once you have included this in your environment, run your Experiment object as follows: - >>> exp.run(num_runs=1, num_steps=1000, convert_to_csv=True) + >>> exp.run(num_runs=1, convert_to_csv=True) After the experiment is complete, look at the "./data" directory. There will be two files, one with the suffix .xml and another with the suffix @@ -51,28 +48,48 @@ class can generate csv files from emission files produced by sumo. These Attributes ---------- + custom_callables : dict < str, lambda > + strings and lambda functions corresponding to some information we want + to extract from the environment. The lambda will be called at each step + to extract information from the env and it will be stored in a dict + keyed by the str. env : flow.envs.Env the environment object the simulator will run """ - def __init__(self, env): - """Instantiate Experiment.""" - self.env = env + def __init__(self, flow_params, custom_callables=None): + """Instantiate the Experiment class. + + Parameters + ---------- + flow_params : dict + flow-specific parameters + custom_callables : dict < str, lambda > + strings and lambda functions corresponding to some information we + want to extract from the environment. The lambda will be called at + each step to extract information from the env and it will be stored + in a dict keyed by the str. + """ + self.custom_callables = custom_callables or {} + + # Get the env name and a creator for the environment. + create_env, _ = make_create_env(flow_params) + + # Create the environment. + self.env = create_env() logging.info(" Starting experiment {} at {}".format( - env.network.name, str(datetime.datetime.utcnow()))) + self.env.network.name, str(datetime.utcnow()))) logging.info("Initializing environment.") - def run(self, num_runs, num_steps, rl_actions=None, convert_to_csv=False): - """Run the given network for a set number of runs and steps per run. + def run(self, num_runs, rl_actions=None, convert_to_csv=False): + """Run the given network for a set number of runs. Parameters ---------- num_runs : int number of runs the experiment should perform - num_steps : int - number of steps to be performs in each run of the experiment rl_actions : method, optional maps states to actions to be performed by the RL agents (if there are any) @@ -82,9 +99,11 @@ def run(self, num_runs, num_steps, rl_actions=None, convert_to_csv=False): Returns ------- - info_dict : dict + info_dict : dict < str, Any > contains returns, average speed per step """ + num_steps = self.env.env_params.horizon + # raise an error if convert_to_csv is set to True but no emission # file will be generated, to avoid getting an error at the end of the # simulation @@ -98,68 +117,69 @@ def run(self, num_runs, num_steps, rl_actions=None, convert_to_csv=False): 'output should be generated. If you do not wish to generate ' 'emissions, set the convert_to_csv parameter to False.') - info_dict = {} + # used to store + info_dict = { + "returns": [], + "velocities": [], + "outflows": [], + } + info_dict.update({ + key: [] for key in self.custom_callables.keys() + }) + if rl_actions is None: def rl_actions(*_): return None - rets = [] - mean_rets = [] - ret_lists = [] - vels = [] - mean_vels = [] - std_vels = [] - outflows = [] + # time profiling information + t = time.time() + times = [] + for i in range(num_runs): - vel = np.zeros(num_steps) - logging.info("Iter #" + str(i)) ret = 0 - ret_list = [] + vel = [] + custom_vals = {key: [] for key in self.custom_callables.keys()} state = self.env.reset() for j in range(num_steps): + t0 = time.time() state, reward, done, _ = self.env.step(rl_actions(state)) - vel[j] = np.mean( - self.env.k.vehicle.get_speed(self.env.k.vehicle.get_ids())) + t1 = time.time() + times.append(1 / (t1 - t0)) + + # Compute the velocity speeds and cumulative returns. + veh_ids = self.env.k.vehicle.get_ids() + vel.append(np.mean(self.env.k.vehicle.get_speed(veh_ids))) ret += reward - ret_list.append(reward) + + # Compute the results for the custom callables. + for (key, lambda_func) in self.custom_callables.items(): + custom_vals[key].append(lambda_func(self.env)) if done: break - rets.append(ret) - vels.append(vel) - mean_rets.append(np.mean(ret_list)) - ret_lists.append(ret_list) - mean_vels.append(np.mean(vel)) - std_vels.append(np.std(vel)) - outflows.append(self.env.k.vehicle.get_outflow_rate(int(500))) - print("Round {0}, return: {1}".format(i, ret)) - - info_dict["returns"] = rets - info_dict["velocities"] = vels - info_dict["mean_returns"] = mean_rets - info_dict["per_step_returns"] = ret_lists - info_dict["mean_outflows"] = np.mean(outflows) - print("Average, std return: {}, {}".format( - np.mean(rets), np.std(rets))) - print("Average, std speed: {}, {}".format( - np.mean(mean_vels), np.std(mean_vels))) - self.env.terminate() + # Store the information from the run in info_dict. + outflow = self.env.k.vehicle.get_outflow_rate(int(500)) + info_dict["returns"].append(ret) + info_dict["velocities"].append(np.mean(vel)) + info_dict["outflows"].append(outflow) + for key in custom_vals.keys(): + info_dict[key].append(np.mean(custom_vals[key])) - if convert_to_csv: - # wait a short period of time to ensure the xml file is readable - time.sleep(0.1) + print("Round {0}, return: {1}".format(i, ret)) - # collect the location of the emission file - dir_path = self.env.sim_params.emission_path - emission_filename = \ - "{0}-emission.xml".format(self.env.network.name) - emission_path = os.path.join(dir_path, emission_filename) + # Save emission data at the end of every rollout. This is skipped + # by the internal method if no emission path was specified. + if self.env.simulator == "traci": + self.env.k.simulation.save_emission(run_id=i) - # convert the emission file into a csv - emission_to_csv(emission_path) + # Print the averages/std for all variables in the info_dict. + for key in info_dict.keys(): + print("Average, std {}: {}, {}".format( + key, np.mean(info_dict[key]), np.std(info_dict[key]))) - # Delete the .xml version of the emission file. - os.remove(emission_path) + print("Total time:", time.time() - t) + print("steps/second:", np.mean(times)) + self.env.terminate() return info_dict diff --git a/flow/core/kernel/network/aimsun.py b/flow/core/kernel/network/aimsun.py index 53b9bbe6a..89971bf58 100644 --- a/flow/core/kernel/network/aimsun.py +++ b/flow/core/kernel/network/aimsun.py @@ -65,7 +65,7 @@ def generate_network(self, network): 'render': self.sim_params.render, "sim_step": self.sim_params.sim_step, "traffic_lights": None, - "scenario_name": self.sim_params.scenario_name, + "network_name": self.sim_params.network_name, "experiment_name": self.sim_params.experiment_name, "replication_name": self.sim_params.replication_name, "centroid_config_name": self.sim_params.centroid_config_name, @@ -81,7 +81,7 @@ def generate_network(self, network): cur_dir = os.path.join(config.PROJECT_PATH, 'flow/core/kernel/network') # TODO: add current time - with open(os.path.join(cur_dir, 'data.json'), 'w') as outfile: + with open(os.path.join(cur_dir, 'data_%s.json' % self.sim_params.port), 'w') as outfile: json.dump(output, outfile, sort_keys=True, indent=4) # path to the Aimsun_Next binary @@ -94,15 +94,17 @@ def generate_network(self, network): # remove network data file if if still exists from # the previous simulation - data_file = 'flow/core/kernel/network/network_data.json' + data_file = 'flow/core/kernel/network/network_data_%s.json' % self.sim_params.port data_file_path = os.path.join(config.PROJECT_PATH, data_file) if os.path.exists(data_file_path): os.remove(data_file_path) - check_file = 'flow/core/kernel/network/network_data_check' + check_file = 'flow/core/kernel/network/network_data_check%s' % self.sim_params.port check_file_path = os.path.join(config.PROJECT_PATH, check_file) if os.path.exists(check_file_path): os.remove(check_file_path) + # we need to make flow directories visible to aimsun's python2.7 + os.environ["PYTHONPATH"] = config.PROJECT_PATH # path to the supplementary file that is used to generate an aimsun # network from a template template_path = network.net_params.template @@ -113,12 +115,14 @@ def generate_network(self, network): script_path = osp.join(config.PROJECT_PATH, 'flow/utils/aimsun/load.py') file_path = osp.join(config.PROJECT_PATH, - 'flow/utils/aimsun/aimsun_template_path') + 'flow/utils/aimsun/aimsun_template_path_%s' % self.sim_params.port) with open(file_path, 'w') as f: - f.write(template_path) + f.write("%s_%s" % (template_path, self.sim_params.port)) + # instances must have unique template paths to avoid crashing? + os.popen('cp %s %s_%s' % (template_path, template_path, self.sim_params.port)) # start the aimsun process - aimsun_call = [aimsun_path, "-script", script_path] + aimsun_call = [aimsun_path, "-script", script_path, str(self.sim_params.port)] self.aimsun_proc = subprocess.Popen(aimsun_call) # merge types into edges @@ -150,10 +154,10 @@ def generate_network(self, network): else: # load network from template - scenar_file = "flow/core/kernel/network/network_data.json" + scenar_file = "flow/core/kernel/network/network_data_%s.json" % self.sim_params.port scenar_path = os.path.join(config.PROJECT_PATH, scenar_file) - check_file = "flow/core/kernel/network/network_data_check" + check_file = "flow/core/kernel/network/network_data_check_%s" % self.sim_params.port check_path = os.path.join(config.PROJECT_PATH, check_file) # a check file is created when all the network data @@ -174,7 +178,7 @@ def generate_network(self, network): # TODO load everything that is in content into the network else: - data_file = 'flow/utils/aimsun/osm_edges.json' + data_file = 'flow/utils/aimsun/osm_edges_%s.json' % self.sim_params.port filepath = os.path.join(config.PROJECT_PATH, data_file) while not os.path.exists(filepath): @@ -257,7 +261,10 @@ def close(self): # delete the json file that was used to read the network data cur_dir = os.path.join(config.PROJECT_PATH, 'flow/core/kernel/network') - os.remove(os.path.join(cur_dir, 'data.json')) + os.remove(os.path.join(cur_dir, 'data_%s.json' % self.sim_params.port)) + if self.network.net_params.template is not None: + os.remove('%s_%s' % (self.network.net_params.template, + self.sim_params.port)) ########################################################################### # State acquisition methods # @@ -276,6 +283,11 @@ def length(self): return sum(self.edge_length(edge_id) for edge_id in self.get_edge_list()) + def non_internal_length(self): + """See parent class.""" + return sum(self.edge_length(edge_id) + for edge_id in self.get_edge_list()) + def speed_limit(self, edge_id): """See parent class.""" try: diff --git a/flow/core/kernel/network/scenario_data_check b/flow/core/kernel/network/scenario_data_check new file mode 100644 index 000000000..e69de29bb diff --git a/flow/core/kernel/network/traci.py b/flow/core/kernel/network/traci.py index af9b1f67d..df57ea1a9 100644 --- a/flow/core/kernel/network/traci.py +++ b/flow/core/kernel/network/traci.py @@ -1,4 +1,5 @@ """Script containing the TraCI network kernel class.""" +import tempfile from flow.core.kernel.network import BaseKernelNetwork from flow.core.util import makexml, printxml, ensure_dir @@ -54,10 +55,8 @@ def __init__(self, master_kernel, sim_params): # directories for the network-specific files that will be generated by # the `generate_network` method - self.net_path = os.path.dirname(os.path.abspath(__file__)) \ - + '/debug/net/' - self.cfg_path = os.path.dirname(os.path.abspath(__file__)) \ - + '/debug/cfg/' + self.net_path = os.path.join(tempfile.gettempdir(), 'flow/debug/net/') + self.cfg_path = os.path.join(tempfile.gettempdir(), 'flow/debug/cfg/') ensure_dir('%s' % self.net_path) ensure_dir('%s' % self.cfg_path) @@ -221,31 +220,28 @@ def close(self): is to prevent them from building up in the debug folder. Note that in the case of import .net.xml files we do not want to delete them. """ + # Those files are being created even if self.network.net_params.template is a path to .net.xml file + files = [self.cfg_path + self.guifn, + self.cfg_path + self.addfn, + self.cfg_path + self.roufn, + self.cfg_path + self.sumfn] + if self.network.net_params.template is None: + files += [self.net_path + self.nodfn, + self.net_path + self.edgfn, + self.net_path + self.cfgfn, + self.net_path + self.confn, + self.net_path + self.typfn, + self.cfg_path + self.netfn] + + for file in files: try: - os.remove(self.net_path + self.nodfn) - os.remove(self.net_path + self.edgfn) - os.remove(self.net_path + self.cfgfn) - os.remove(self.cfg_path + self.addfn) - os.remove(self.cfg_path + self.guifn) - os.remove(self.cfg_path + self.netfn) - os.remove(self.cfg_path + self.roufn) - os.remove(self.cfg_path + self.sumfn) - except FileNotFoundError: + os.remove(file) + except (FileNotFoundError, OSError): # the files were never created - pass - - # the connection file is not always created - try: - os.remove(self.net_path + self.confn) - except OSError: - pass - - # neither is the type file - try: - os.remove(self.net_path + self.typfn) - except OSError: - pass + # the connection file is not always created + # neither is the type file + continue def get_edge(self, x): """See parent class.""" @@ -515,6 +511,7 @@ def generate_net(self, ' --output-file=' + self.cfg_path + self.netfn + ' --no-internal-links="false"' ], + stdout=subprocess.DEVNULL, shell=True) # collect data from the generated network configuration file diff --git a/flow/core/kernel/simulation/aimsun.py b/flow/core/kernel/simulation/aimsun.py index ef4ed0b24..91b4c4abd 100644 --- a/flow/core/kernel/simulation/aimsun.py +++ b/flow/core/kernel/simulation/aimsun.py @@ -61,9 +61,6 @@ def start_simulation(self, network, sim_params): a simulation, and creates a class to communicate with the simulation via an TCP connection. """ - # FIXME: hack - sim_params.port = 9999 - # save the simulation step size (for later use) self.sim_step = sim_params.sim_step diff --git a/flow/core/kernel/simulation/traci.py b/flow/core/kernel/simulation/traci.py index 108d706c0..20643ee5b 100644 --- a/flow/core/kernel/simulation/traci.py +++ b/flow/core/kernel/simulation/traci.py @@ -11,6 +11,7 @@ import logging import subprocess import signal +import csv # Number of retries on restarting SUMO before giving up @@ -21,6 +22,32 @@ class TraCISimulation(KernelSimulation): """Sumo simulation kernel. Extends flow.core.kernel.simulation.KernelSimulation + + Attributes + ---------- + sumo_proc : subprocess.Popen + contains the subprocess.Popen instance used to start traci + sim_step : float + seconds per simulation step + emission_path : str or None + Path to the folder in which to create the emissions output. Emissions + output is not generated if this value is not specified + time : float + used to internally keep track of the simulation time + stored_data : dict >> + a dict object used to store additional data if an emission file is + provided. The first key corresponds to the name of the vehicle, the + second corresponds to the time the sample was issued, and the final + keys represent the additional data stored at every given time for every + vehicle, and consists of the following keys: + + * acceleration (no noise): the accelerations issued to the vehicle, + excluding noise + * acceleration (requested): the requested acceleration by the vehicle, + including noise + * acceleration (actual): the actual acceleration by the vehicle, + collected by computing the difference between the speeds of the + vehicle and dividing it by the sim_step term """ def __init__(self, master_kernel): @@ -33,8 +60,12 @@ def __init__(self, master_kernel): sub-kernels) """ KernelSimulation.__init__(self, master_kernel) - # contains the subprocess.Popen instance used to start traci + self.sumo_proc = None + self.sim_step = None + self.emission_path = None + self.time = 0 + self.stored_data = dict() def pass_api(self, kernel_api): """See parent class. @@ -46,9 +77,14 @@ def pass_api(self, kernel_api): # subscribe some simulation parameters needed to check for entering, # exiting, and colliding vehicles self.kernel_api.simulation.subscribe([ - tc.VAR_DEPARTED_VEHICLES_IDS, tc.VAR_ARRIVED_VEHICLES_IDS, - tc.VAR_TELEPORT_STARTING_VEHICLES_IDS, tc.VAR_TIME_STEP, - tc.VAR_DELTA_T + tc.VAR_DEPARTED_VEHICLES_IDS, + tc.VAR_ARRIVED_VEHICLES_IDS, + tc.VAR_TELEPORT_STARTING_VEHICLES_IDS, + tc.VAR_TIME_STEP, + tc.VAR_DELTA_T, + tc.VAR_LOADED_VEHICLES_NUMBER, + tc.VAR_DEPARTED_VEHICLES_NUMBER, + tc.VAR_ARRIVED_VEHICLES_NUMBER ]) def simulation_step(self): @@ -57,10 +93,61 @@ def simulation_step(self): def update(self, reset): """See parent class.""" - pass + if reset: + self.time = 0 + else: + self.time += self.sim_step + + # Collect the additional data to store in the emission file. + if self.emission_path is not None: + kv = self.master_kernel.vehicle + for veh_id in self.master_kernel.vehicle.get_ids(): + t = round(self.time, 2) + + # some miscellaneous pre-processing + position = kv.get_2d_position(veh_id) + + # Make sure dictionaries corresponding to the vehicle and + # time are available. + if veh_id not in self.stored_data.keys(): + self.stored_data[veh_id] = dict() + if t not in self.stored_data[veh_id].keys(): + self.stored_data[veh_id][t] = dict() + + # Add the speed, position, and lane data. + self.stored_data[veh_id][t].update({ + "speed": kv.get_speed(veh_id), + "lane_number": kv.get_lane(veh_id), + "edge_id": kv.get_edge(veh_id), + "relative_position": kv.get_position(veh_id), + "x": position[0], + "y": position[1], + "headway": kv.get_headway(veh_id), + "leader_id": kv.get_leader(veh_id), + "follower_id": kv.get_follower(veh_id), + "leader_rel_speed": + kv.get_speed(kv.get_leader(veh_id)) + - kv.get_speed(veh_id), + "target_accel_with_noise_with_failsafe": + kv.get_accel(veh_id, noise=True, failsafe=True), + "target_accel_no_noise_no_failsafe": + kv.get_accel(veh_id, noise=False, failsafe=False), + "target_accel_with_noise_no_failsafe": + kv.get_accel(veh_id, noise=True, failsafe=False), + "target_accel_no_noise_with_failsafe": + kv.get_accel(veh_id, noise=False, failsafe=True), + "realized_accel": + kv.get_realized_accel(veh_id), + "road_grade": kv.get_road_grade(veh_id), + "distance": kv.get_distance(veh_id), + }) def close(self): """See parent class.""" + # Save the emission data to a csv. + if self.emission_path is not None: + self.save_emission() + self.kernel_api.close() def check_collision(self): @@ -70,10 +157,24 @@ def check_collision(self): def start_simulation(self, network, sim_params): """Start a sumo simulation instance. - This method uses the configuration files created by the network class - to initialize a sumo instance. Also initializes a traci connection to - interface with sumo from Python. + This method performs the following operations: + + 1. It collect the simulation step size and the emission path + information. If an emission path is specifies, it ensures that the + path exists. + 2. It also uses the configuration files created by the network class to + initialize a sumo instance. + 3. Finally, It initializes a traci connection to interface with sumo + from Python and returns the connection. """ + # Save the simulation step size (for later use). + self.sim_step = sim_params.sim_step + + # Update the emission path term. + self.emission_path = sim_params.emission_path + if self.emission_path is not None: + ensure_dir(self.emission_path) + error = None for _ in range(RETRIES_ON_ERROR): try: @@ -91,7 +192,11 @@ def start_simulation(self, network, sim_params): "--step-length", str(sim_params.sim_step) ] - # add step logs (if requested) + # use a ballistic integration step (if request) + if sim_params.use_ballistic: + sumo_call.append("--step-method.ballistic") + + # ignore step logs (if requested) if sim_params.no_step_log: sumo_call.append("--no-step-log") @@ -100,17 +205,6 @@ def start_simulation(self, network, sim_params): sumo_call.append("--lateral-resolution") sumo_call.append(str(sim_params.lateral_resolution)) - # add the emission path to the sumo command (if requested) - if sim_params.emission_path is not None: - ensure_dir(sim_params.emission_path) - emission_out = os.path.join( - sim_params.emission_path, - "{0}-emission.xml".format(network.name)) - sumo_call.append("--emission-output") - sumo_call.append(emission_out) - else: - emission_out = None - if sim_params.overtake_right: sumo_call.append("--lanechange.overtake-right") sumo_call.append("true") @@ -137,12 +231,14 @@ def start_simulation(self, network, sim_params): if sim_params.num_clients > 1: logging.info(" Num clients are" + str(sim_params.num_clients)) - logging.debug(" Emission file: " + str(emission_out)) + logging.debug(" Emission file: " + str(self.emission_path)) logging.debug(" Step length: " + str(sim_params.sim_step)) # Opening the I/O thread to SUMO self.sumo_proc = subprocess.Popen( - sumo_call, preexec_fn=os.setsid) + sumo_call, + stdout=subprocess.DEVNULL + ) # wait a small period of time for the subprocess to activate # before trying to connect with traci @@ -168,3 +264,66 @@ def teardown_sumo(self): os.killpg(self.sumo_proc.pid, signal.SIGTERM) except Exception as e: print("Error during teardown: {}".format(e)) + + def save_emission(self, run_id=0): + """Save any collected emission data to a csv file. + + If not data was collected, nothing happens. Moreover, any internally + stored data by this class is clear whenever data is stored. + + Parameters + ---------- + run_id : int + the rollout number, appended to the name of the emission file. Used + to store emission files from multiple rollouts run sequentially. + """ + # If there is no stored data, ignore this operation. This is to ensure + # that data isn't deleted if the operation is called twice. + if len(self.stored_data) == 0: + return + + # Get a csv name for the emission file. + name = "{}-{}_emission.csv".format( + self.master_kernel.network.network.name, run_id) + + # The name of all stored data-points (excluding id and time) + stored_ids = [ + "x", + "y", + "speed", + "headway", + "leader_id", + "target_accel_with_noise_with_failsafe", + "target_accel_no_noise_no_failsafe", + "target_accel_with_noise_no_failsafe", + "target_accel_no_noise_with_failsafe", + "realized_accel", + "road_grade", + "edge_id", + "lane_number", + "distance", + "relative_position", + "follower_id", + "leader_rel_speed", + ] + + # Update the stored data to push to the csv file. + final_data = {"time": [], "id": []} + final_data.update({key: [] for key in stored_ids}) + + for veh_id in self.stored_data.keys(): + for t in self.stored_data[veh_id].keys(): + final_data['time'].append(t) + final_data['id'].append(veh_id) + for key in stored_ids: + final_data[key].append(self.stored_data[veh_id][t][key]) + + with open(os.path.join(self.emission_path, name), "w") as f: + print(os.path.join(self.emission_path, name), self.emission_path) + writer = csv.writer(f, delimiter=',') + writer.writerow(final_data.keys()) + writer.writerows(zip(*final_data.values())) + + # Clear all memory from the stored data. This is useful if this + # function is called in between resets. + self.stored_data.clear() diff --git a/flow/core/kernel/vehicle/aimsun.py b/flow/core/kernel/vehicle/aimsun.py index 9e791a10d..16c94558a 100644 --- a/flow/core/kernel/vehicle/aimsun.py +++ b/flow/core/kernel/vehicle/aimsun.py @@ -65,6 +65,7 @@ def __init__(self, # number of vehicles to exit the network for every time-step self._num_arrived = [] self._arrived_ids = [] + self._arrived_rl_ids = [] # contains conversion from Flow-ID to Aimsun-ID self._id_aimsun2flow = {} @@ -174,11 +175,17 @@ def update(self, reset): added_vehicles = self.kernel_api.get_entered_ids() exited_vehicles = self.kernel_api.get_exited_ids() + # keep track of arrived rl vehicles + arrived_rl_ids = [] + # add the new vehicles if they should be tracked for aimsun_id in added_vehicles: veh_type = self.kernel_api.get_vehicle_type_name(aimsun_id) if veh_type in self.tracked_vehicle_types: self._add_departed(aimsun_id) + if aimsun_id in self.get_rl_ids(): + arrived_rl_ids.append(aimsun_id) + self._arrived_rl_ids.append(arrived_rl_ids) # remove the exited vehicles if they were tracked if not reset: @@ -401,6 +408,10 @@ def add(self, veh_id, type_id, edge, pos, lane, speed): self.num_type[type_id] = 1 self.total_num_type[type_id] = 1 + def reset(self): + """See parent class.""" + raise NotImplementedError + def remove(self, aimsun_id): """See parent class.""" veh_id = self._id_aimsun2flow[aimsun_id] @@ -513,7 +524,7 @@ def choose_routes(self, veh_id, route_choices): edge the vehicle is currently on. If a value of None is provided, the vehicle does not update its route """ - pass # FIXME + raise NotImplementedError # FIXME # for i, veh_id in enumerate(veh_ids): # if route_choices[i] is not None: # aimsun_id = self._id_flow2aimsun[veh_id] @@ -521,6 +532,10 @@ def choose_routes(self, veh_id, route_choices): # self.kernel_api.AKIVehTrackedModifyNextSections( # aimsun_id, size_next_sections, route_choices[i]) + def set_max_speed(self, veh_id, max_speed): + """See parent class.""" + raise NotImplementedError + ########################################################################### # Methods to visually distinguish vehicles by {RL, observed, unobserved} # ########################################################################### @@ -556,10 +571,30 @@ def get_observed_ids(self): """Return the list of observed vehicles.""" return self.__observed_ids + def get_color(self, veh_id): + """See parent class.""" + raise NotImplementedError + + def set_color(self, veh_id, color): + """See parent class.""" + raise NotImplementedError + ########################################################################### # State acquisition methods # ########################################################################### + def get_orientation(self, veh_id): + """See parent class.""" + raise NotImplementedError + + def get_timestep(self, veh_id): + """See parent class.""" + raise NotImplementedError + + def get_timedelta(self, veh_id): + """See parent class.""" + raise NotImplementedError + def get_ids(self): """See parent class.""" return self.__ids @@ -607,6 +642,32 @@ def get_num_arrived(self): else: return 0 + def get_arrived_ids(self): + """See parent class.""" + raise NotImplementedError + + def get_arrived_rl_ids(self, k=1): + """See parent class.""" + if len(self._arrived_rl_ids) > 0: + arrived = [] + for arr in self._arrived_rl_ids[-k:]: + arrived.extend(arr) + return arrived + else: + return 0 + + def get_departed_ids(self): + """See parent class.""" + raise NotImplementedError + + def get_num_not_departed(self): + """See parent class.""" + raise NotImplementedError + + def get_fuel_consumption(self): + """See parent class.""" + raise NotImplementedError + def get_type(self, veh_id): """See parent class.""" if isinstance(veh_id, (list, np.ndarray)): @@ -623,6 +684,10 @@ def get_speed(self, veh_id, error=-1001): return [self.get_speed(veh, error) for veh in veh_id] return self.__vehicles[veh_id]['tracking_info'].CurrentSpeed / 3.6 + def get_default_speed(self, veh_id, error=-1001): + """See parent class.""" + raise NotImplementedError + def get_position(self, veh_id, error=-1001): """See parent class.""" if isinstance(veh_id, (list, np.ndarray)): @@ -802,3 +867,7 @@ def get_lane_followers_speed(self, veh_id, error=None): def get_lane_leaders_speed(self, veh_id, error=None): """See parent class.""" raise NotImplementedError + + def get_max_speed(self, veh_id, error): + """See parent class.""" + raise NotImplementedError diff --git a/flow/core/kernel/vehicle/base.py b/flow/core/kernel/vehicle/base.py index 59f59617e..a433b8924 100644 --- a/flow/core/kernel/vehicle/base.py +++ b/flow/core/kernel/vehicle/base.py @@ -1,7 +1,9 @@ """Script containing the base vehicle kernel class.""" +from abc import ABCMeta, abstractmethod -class KernelVehicle(object): + +class KernelVehicle(object, metaclass=ABCMeta): """Flow vehicle kernel. This kernel sub-class is used to interact with the simulator with regards @@ -66,6 +68,7 @@ def pass_api(self, kernel_api): # Methods for interacting with the simulator # ########################################################################### + @abstractmethod def update(self, reset): """Update the vehicle kernel with data from the current time step. @@ -78,8 +81,9 @@ def update(self, reset): specifies whether the simulator was reset in the last simulation step """ - raise NotImplementedError + pass + @abstractmethod def add(self, veh_id, type_id, edge, pos, lane, speed): """Add a vehicle to the network. @@ -98,8 +102,14 @@ def add(self, veh_id, type_id, edge, pos, lane, speed): speed : float starting speed of the added vehicle """ - raise NotImplementedError + pass + + @abstractmethod + def reset(self): + """Reset any additional state that needs to be reset.""" + pass + @abstractmethod def remove(self, veh_id): """Remove a vehicle. @@ -115,20 +125,26 @@ def remove(self, veh_id): veh_id : str unique identifier of the vehicle to be removed """ - raise NotImplementedError + pass - def apply_acceleration(self, veh_id, acc): + @abstractmethod + def apply_acceleration(self, veh_id, acc, smooth=True): """Apply the acceleration requested by a vehicle in the simulator. + In SUMO, this function applies slowDown method which applies smoothing. + Parameters ---------- veh_id : str or list of str list of vehicle identifiers acc : float or array_like requested accelerations from the vehicles + smooth : bool + whether to apply acceleration smoothly or not, default: True """ - raise NotImplementedError + pass + @abstractmethod def apply_lane_change(self, veh_id, direction): """Apply an instantaneous lane-change to a set of vehicles. @@ -151,8 +167,9 @@ def apply_lane_change(self, veh_id, direction): ValueError If any of the direction values are not -1, 0, or 1. """ - raise NotImplementedError + pass + @abstractmethod def choose_routes(self, veh_id, route_choices): """Update the route choice of vehicles in the network. @@ -165,8 +182,9 @@ def choose_routes(self, veh_id, route_choices): edge the vehicle is currently on. If a value of None is provided, the vehicle does not update its route """ - raise NotImplementedError + pass + @abstractmethod def set_max_speed(self, veh_id, max_speed): """Update the maximum allowable speed by a vehicles in the network. @@ -177,115 +195,162 @@ def set_max_speed(self, veh_id, max_speed): max_speed : float desired max speed by the vehicle """ - raise NotImplementedError + pass ########################################################################### # Methods to visually distinguish vehicles by {RL, observed, unobserved} # ########################################################################### + @abstractmethod def update_vehicle_colors(self): """Modify the color of vehicles if rendering is active.""" - raise NotImplementedError + pass + @abstractmethod def set_observed(self, veh_id): """Add a vehicle to the list of observed vehicles.""" - raise NotImplementedError + pass + @abstractmethod def remove_observed(self, veh_id): """Remove a vehicle from the list of observed vehicles.""" - raise NotImplementedError + pass + @abstractmethod def get_observed_ids(self): """Return the list of observed vehicles.""" - raise NotImplementedError + pass + @abstractmethod def get_color(self, veh_id): """Return and RGB tuple of the color of the specified vehicle.""" - raise NotImplementedError + pass + @abstractmethod def set_color(self, veh_id, color): """Set the color of the specified vehicle with the RGB tuple.""" - raise NotImplementedError + pass ########################################################################### # State acquisition methods # ########################################################################### + @abstractmethod def get_orientation(self, veh_id): """Return the orientation of the vehicle of veh_id.""" - raise NotImplementedError + pass + @abstractmethod def get_timestep(self, veh_id): """Return the time step of the vehicle of veh_id.""" - raise NotImplementedError + pass + @abstractmethod def get_timedelta(self, veh_id): """Return the simulation time delta of the vehicle of veh_id.""" - raise NotImplementedError + pass + @abstractmethod def get_type(self, veh_id): """Return the type of the vehicle of veh_id.""" - raise NotImplementedError + pass + @abstractmethod def get_ids(self): """Return the names of all vehicles currently in the network.""" - raise NotImplementedError + pass + @abstractmethod def get_human_ids(self): """Return the names of all non-rl vehicles currently in the network.""" - raise NotImplementedError + pass + @abstractmethod def get_controlled_ids(self): """Return the names of all flow acceleration-controlled vehicles. This only include vehicles that are currently in the network. """ - raise NotImplementedError + pass + @abstractmethod def get_controlled_lc_ids(self): """Return the names of all flow lane change-controlled vehicles. This only include vehicles that are currently in the network. """ - raise NotImplementedError + pass + @abstractmethod def get_rl_ids(self): """Return the names of all rl-controlled vehicles in the network.""" - raise NotImplementedError + pass + @abstractmethod def get_ids_by_edge(self, edges): """Return the names of all vehicles in the specified edge. If no vehicles are currently in the edge, then returns an empty list. """ - raise NotImplementedError + pass + @abstractmethod def get_inflow_rate(self, time_span): """Return the inflow rate (in veh/hr) of vehicles from the network. This value is computed over the specified **time_span** seconds. """ - raise NotImplementedError + pass + @abstractmethod def get_outflow_rate(self, time_span): """Return the outflow rate (in veh/hr) of vehicles from the network. This value is computed over the specified **time_span** seconds. """ - raise NotImplementedError + pass + @abstractmethod def get_num_arrived(self): """Return the number of vehicles that arrived in the last time step.""" - raise NotImplementedError + pass + @abstractmethod def get_arrived_ids(self): """Return the ids of vehicles that arrived in the last time step.""" - raise NotImplementedError + pass + @abstractmethod def get_departed_ids(self): """Return the ids of vehicles that departed in the last time step.""" - raise NotImplementedError + pass + + @abstractmethod + def get_num_not_departed(self): + """Return the number of vehicles not departed in the last time step. + + This includes vehicles that were loaded but not departed. + """ + pass + @abstractmethod + def get_fuel_consumption(self, veh_id, error=-1001): + """Return the mpg / s of the specified vehicle. + + Parameters + ---------- + veh_id : str or list of str + vehicle id, or list of vehicle ids + error : any, optional + value that is returned if the vehicle is not found + + Returns + ------- + float + """ + pass + + @abstractmethod def get_speed(self, veh_id, error=-1001): """Return the speed of the specified vehicle. @@ -300,8 +365,9 @@ def get_speed(self, veh_id, error=-1001): ------- float """ - raise NotImplementedError + pass + @abstractmethod def get_default_speed(self, veh_id, error=-1001): """Return the expected speed if no control were applied. @@ -316,8 +382,9 @@ def get_default_speed(self, veh_id, error=-1001): ------- float """ - raise NotImplementedError + pass + @abstractmethod def get_position(self, veh_id, error=-1001): """Return the position of the vehicle relative to its current edge. @@ -332,8 +399,9 @@ def get_position(self, veh_id, error=-1001): ------- float """ - raise NotImplementedError + pass + @abstractmethod def get_edge(self, veh_id, error=""): """Return the edge the specified vehicle is currently on. @@ -348,8 +416,9 @@ def get_edge(self, veh_id, error=""): ------- str """ - raise NotImplementedError + pass + @abstractmethod def get_lane(self, veh_id, error=-1001): """Return the lane index of the specified vehicle. @@ -364,8 +433,9 @@ def get_lane(self, veh_id, error=-1001): ------- int """ - raise NotImplementedError + pass + @abstractmethod def get_route(self, veh_id, error=list()): """Return the route of the specified vehicle. @@ -380,8 +450,9 @@ def get_route(self, veh_id, error=list()): ------- list of str """ - raise NotImplementedError + pass + @abstractmethod def get_length(self, veh_id, error=-1001): """Return the length of the specified vehicle. @@ -396,8 +467,9 @@ def get_length(self, veh_id, error=-1001): ------- float """ - raise NotImplementedError + pass + @abstractmethod def get_leader(self, veh_id, error=""): """Return the leader of the specified vehicle. @@ -412,8 +484,9 @@ def get_leader(self, veh_id, error=""): ------- str """ - raise NotImplementedError + pass + @abstractmethod def get_follower(self, veh_id, error=""): """Return the follower of the specified vehicle. @@ -428,8 +501,9 @@ def get_follower(self, veh_id, error=""): ------- str """ - raise NotImplementedError + pass + @abstractmethod def get_headway(self, veh_id, error=-1001): """Return the headway of the specified vehicle(s). @@ -444,8 +518,9 @@ def get_headway(self, veh_id, error=-1001): ------- float """ - raise NotImplementedError + pass + @abstractmethod def get_last_lc(self, veh_id, error=-1001): """Return the last time step a vehicle changed lanes. @@ -464,8 +539,9 @@ def get_last_lc(self, veh_id, error=-1001): ------- int """ - raise NotImplementedError + pass + @abstractmethod def get_acc_controller(self, veh_id, error=None): """Return the acceleration controller of the specified vehicle. @@ -480,8 +556,9 @@ def get_acc_controller(self, veh_id, error=None): ------- object """ - raise NotImplementedError + pass + @abstractmethod def get_lane_changing_controller(self, veh_id, error=None): """Return the lane changing controller of the specified vehicle. @@ -496,8 +573,9 @@ def get_lane_changing_controller(self, veh_id, error=None): ------- object """ - raise NotImplementedError + pass + @abstractmethod def get_routing_controller(self, veh_id, error=None): """Return the routing controller of the specified vehicle. @@ -512,8 +590,9 @@ def get_routing_controller(self, veh_id, error=None): ------- object """ - raise NotImplementedError + pass + @abstractmethod def get_lane_headways(self, veh_id, error=list()): """Return the lane headways of the specified vehicles. @@ -531,8 +610,9 @@ def get_lane_headways(self, veh_id, error=list()): ------- list of float """ - raise NotImplementedError + pass + @abstractmethod def get_lane_leaders_speed(self, veh_id, error=list()): """Return the speed of the leaders of the specified vehicles. @@ -552,8 +632,9 @@ def get_lane_leaders_speed(self, veh_id, error=list()): ------- list of float """ - raise NotImplementedError + pass + @abstractmethod def get_lane_followers_speed(self, veh_id, error=list()): """Return the speed of the followers of the specified vehicles. @@ -573,8 +654,9 @@ def get_lane_followers_speed(self, veh_id, error=list()): ------- list of float """ - raise NotImplementedError + pass + @abstractmethod def get_lane_leaders(self, veh_id, error=list()): """Return the leaders for the specified vehicle in all lanes. @@ -589,8 +671,9 @@ def get_lane_leaders(self, veh_id, error=list()): ------- lis of str """ - raise NotImplementedError + pass + @abstractmethod def get_lane_tailways(self, veh_id, error=list()): """Return the lane tailways of the specified vehicle. @@ -608,8 +691,9 @@ def get_lane_tailways(self, veh_id, error=list()): ------- list of float """ - raise NotImplementedError + pass + @abstractmethod def get_lane_followers(self, veh_id, error=list()): """Return the followers for the specified vehicle in all lanes. @@ -624,8 +708,9 @@ def get_lane_followers(self, veh_id, error=list()): ------- list of str """ - raise NotImplementedError + pass + @abstractmethod def get_x_by_id(self, veh_id): """Provide a 1-D representation of the position of a vehicle. @@ -642,8 +727,9 @@ def get_x_by_id(self, veh_id): ------- float """ - raise NotImplementedError + pass + @abstractmethod def get_max_speed(self, veh_id, error): """Return the max speed of the specified vehicle. @@ -658,4 +744,33 @@ def get_max_speed(self, veh_id, error): ------- float """ - raise NotImplementedError + pass + + ########################################################################### + # Methods for Datapipeline # + ########################################################################### + + @abstractmethod + def get_accel(self, veh_id, noise=True, failsafe=True): + """Return the acceleration of vehicle with veh_id.""" + pass + + @abstractmethod + def update_accel(self, veh_id, accel, noise=True, failsafe=True): + """Update stored acceleration of vehicle with veh_id.""" + pass + + @abstractmethod + def get_2d_position(self, veh_id, error=-1001): + """Return (x, y) position of vehicle with veh_id.""" + pass + + @abstractmethod + def get_realized_accel(self, veh_id): + """Return the acceleration that the vehicle actually make.""" + pass + + @abstractmethod + def get_road_grade(self, veh_id): + """Return the road-grade of the vehicle with veh_id.""" + pass diff --git a/flow/core/kernel/vehicle/traci.py b/flow/core/kernel/vehicle/traci.py index 0ed8ed220..39bfb35da 100644 --- a/flow/core/kernel/vehicle/traci.py +++ b/flow/core/kernel/vehicle/traci.py @@ -18,6 +18,12 @@ WHITE = (255, 255, 255) CYAN = (0, 255, 255) RED = (255, 0, 0) +GREEN = (0, 255, 0) +STEPS = 10 +rdelta = 255 / STEPS +# smoothly go from red to green as the speed increases +color_bins = [[int(255 - rdelta * i), int(rdelta * i), 0] for i in + range(STEPS + 1)] class TraCIVehicle(KernelVehicle): @@ -51,6 +57,8 @@ def __init__(self, self.num_vehicles = 0 # number of rl vehicles in the network self.num_rl_vehicles = 0 + # number of vehicles loaded but not departed vehicles + self.num_not_departed = 0 # contains the parameters associated with each type of vehicle self.type_parameters = {} @@ -63,17 +71,22 @@ def __init__(self, # number of vehicles that entered the network for every time-step self._num_departed = [] - self._departed_ids = [] + self._departed_ids = 0 # number of vehicles to exit the network for every time-step self._num_arrived = [] - self._arrived_ids = [] + self._arrived_ids = 0 + self._arrived_rl_ids = [] # whether or not to automatically color vehicles try: - self._color_vehicles = sim_params.color_vehicles + self._color_by_speed = sim_params.color_by_speed + self._force_color_update = sim_params.force_color_update except AttributeError: - self._color_vehicles = False + self._force_color_update = False + + # old speeds used to compute accelerations + self.previous_speeds = {} def initialize(self, vehicles): """Initialize vehicle state information. @@ -91,6 +104,7 @@ def initialize(self, vehicles): self.minGap = vehicles.minGap self.num_vehicles = 0 self.num_rl_vehicles = 0 + self.num_not_departed = 0 self.__vehicles.clear() for typ in vehicles.initial: @@ -120,14 +134,20 @@ def update(self, reset): specifies whether the simulator was reset in the last simulation step """ + # copy over the previous speeds + vehicle_obs = {} for veh_id in self.__ids: + self.previous_speeds[veh_id] = self.get_speed(veh_id) vehicle_obs[veh_id] = \ self.kernel_api.vehicle.getSubscriptionResults(veh_id) sim_obs = self.kernel_api.simulation.getSubscriptionResults() + arrived_rl_ids = [] # remove exiting vehicles from the vehicles class for veh_id in sim_obs[tc.VAR_ARRIVED_VEHICLES_IDS]: + if veh_id in self.get_rl_ids(): + arrived_rl_ids.append(veh_id) if veh_id in sim_obs[tc.VAR_TELEPORT_STARTING_VEHICLES_IDS]: # this is meant to resolve the KeyError bug when there are # collisions @@ -137,6 +157,7 @@ def update(self, reset): # haven't been removed already if vehicle_obs[veh_id] is None: vehicle_obs.pop(veh_id, None) + self._arrived_rl_ids.append(arrived_rl_ids) # add entering vehicles into the vehicles class for veh_id in sim_obs[tc.VAR_DEPARTED_VEHICLES_IDS]: @@ -163,13 +184,15 @@ def update(self, reset): self.prev_last_lc[veh_id] = -float("inf") self._num_departed.clear() self._num_arrived.clear() - self._departed_ids.clear() - self._arrived_ids.clear() + self._departed_ids = 0 + self._arrived_ids = 0 + self._arrived_rl_ids.clear() + self.num_not_departed = 0 # add vehicles from a network template, if applicable if hasattr(self.master_kernel.network.network, "template_vehicles"): - for veh_id in self.master_kernel.network.network.\ + for veh_id in self.master_kernel.network.network. \ template_vehicles: vals = deepcopy(self.master_kernel.network.network. template_vehicles[veh_id]) @@ -188,11 +211,14 @@ def update(self, reset): self.__vehicles[veh_id]["last_lc"] = self.time_counter # updated the list of departed and arrived vehicles - self._num_departed.append( - len(sim_obs[tc.VAR_DEPARTED_VEHICLES_IDS])) - self._num_arrived.append(len(sim_obs[tc.VAR_ARRIVED_VEHICLES_IDS])) - self._departed_ids.append(sim_obs[tc.VAR_DEPARTED_VEHICLES_IDS]) - self._arrived_ids.append(sim_obs[tc.VAR_ARRIVED_VEHICLES_IDS]) + self._num_departed.append(sim_obs[tc.VAR_LOADED_VEHICLES_NUMBER]) + self._num_arrived.append(sim_obs[tc.VAR_ARRIVED_VEHICLES_NUMBER]) + self._departed_ids = sim_obs[tc.VAR_DEPARTED_VEHICLES_IDS] + self._arrived_ids = sim_obs[tc.VAR_ARRIVED_VEHICLES_IDS] + + # update the number of not departed vehicles + self.num_not_departed += sim_obs[tc.VAR_LOADED_VEHICLES_NUMBER] - \ + sim_obs[tc.VAR_DEPARTED_VEHICLES_NUMBER] # update the "headway", "leader", and "follower" variables for veh_id in self.__ids: @@ -293,7 +319,6 @@ def _add_departed(self, veh_id, veh_type): if accel_controller[0] == RLController: if veh_id not in self.__rl_ids: self.__rl_ids.append(veh_id) - self.num_rl_vehicles += 1 else: if veh_id not in self.__human_ids: self.__human_ids.append(veh_id) @@ -304,9 +329,15 @@ def _add_departed(self, veh_id, veh_type): # subscribe the new vehicle self.kernel_api.vehicle.subscribe(veh_id, [ - tc.VAR_LANE_INDEX, tc.VAR_LANEPOSITION, tc.VAR_ROAD_ID, - tc.VAR_SPEED, tc.VAR_EDGES, tc.VAR_POSITION, tc.VAR_ANGLE, - tc.VAR_SPEED_WITHOUT_TRACI + tc.VAR_LANE_INDEX, tc.VAR_LANEPOSITION, + tc.VAR_ROAD_ID, + tc.VAR_SPEED, + tc.VAR_EDGES, + tc.VAR_POSITION, + tc.VAR_ANGLE, + tc.VAR_SPEED_WITHOUT_TRACI, + tc.VAR_FUELCONSUMPTION, + tc.VAR_DISTANCE ]) self.kernel_api.vehicle.subscribeLeader(veh_id, 2000) @@ -341,15 +372,22 @@ def _add_departed(self, veh_id, veh_type): self.kernel_api.vehicle.getLaneIndex(veh_id) self.__sumo_obs[veh_id][tc.VAR_SPEED] = \ self.kernel_api.vehicle.getSpeed(veh_id) + self.__sumo_obs[veh_id][tc.VAR_FUELCONSUMPTION] = \ + self.kernel_api.vehicle.getFuelConsumption(veh_id) # make sure that the order of rl_ids is kept sorted self.__rl_ids.sort() + self.num_rl_vehicles = len(self.__rl_ids) # get the subscription results from the new vehicle new_obs = self.kernel_api.vehicle.getSubscriptionResults(veh_id) return new_obs + def reset(self): + """See parent class.""" + self.previous_speeds = {} + def remove(self, veh_id): """See parent class.""" # remove from sumo @@ -482,17 +520,38 @@ def get_num_arrived(self): def get_arrived_ids(self): """See parent class.""" - if len(self._arrived_ids) > 0: - return self._arrived_ids[-1] + return self._arrived_ids + + def get_arrived_rl_ids(self, k=1): + """See parent class.""" + if len(self._arrived_rl_ids) > 0: + arrived = [] + for arr in self._arrived_rl_ids[-k:]: + arrived.extend(arr) + return arrived else: return 0 def get_departed_ids(self): """See parent class.""" - if len(self._departed_ids) > 0: - return self._departed_ids[-1] - else: - return 0 + return self._departed_ids + + def get_num_not_departed(self): + """See parent class.""" + return self.num_not_departed + + def get_fuel_consumption(self, veh_id, error=-1001): + """Return fuel consumption in gallons/s.""" + ml_to_gallons = 0.000264172 + if isinstance(veh_id, (list, np.ndarray)): + return [self.get_fuel_consumption(vehID, error) for vehID in veh_id] + return self.__sumo_obs.get(veh_id, {}).get(tc.VAR_FUELCONSUMPTION, error) * ml_to_gallons + + def get_previous_speed(self, veh_id, error=-1001): + """See parent class.""" + if isinstance(veh_id, (list, np.ndarray)): + return [self.get_previous_speed(vehID, error) for vehID in veh_id] + return self.previous_speeds.get(veh_id, 0) def get_speed(self, veh_id, error=-1001): """See parent class.""" @@ -894,18 +953,22 @@ def _prev_edge_followers(self, veh_id, edge_dict, lane, num_edges): return tailway, follower - def apply_acceleration(self, veh_ids, acc): + def apply_acceleration(self, veh_ids, acc, smooth=True): """See parent class.""" - # to hand the case of a single vehicle + # to handle the case of a single vehicle if type(veh_ids) == str: veh_ids = [veh_ids] acc = [acc] for i, vid in enumerate(veh_ids): if acc[i] is not None and vid in self.get_ids(): + self.__vehicles[vid]["accel"] = acc[i] this_vel = self.get_speed(vid) next_vel = max([this_vel + acc[i] * self.sim_step, 0]) - self.kernel_api.vehicle.slowDown(vid, next_vel, 1e-3) + if smooth: + self.kernel_api.vehicle.slowDown(vid, next_vel, 1e-3) + else: + self.kernel_api.vehicle.setSpeed(vid, next_vel) def apply_lane_change(self, veh_ids, direction): """See parent class.""" @@ -935,7 +998,7 @@ def apply_lane_change(self, veh_ids, direction): # perform the requested lane action action in TraCI if target_lane != this_lane: self.kernel_api.vehicle.changeLane( - veh_id, int(target_lane), 100000) + veh_id, int(target_lane), self.sim_step) if veh_id in self.get_rl_ids(): self.prev_last_lc[veh_id] = \ @@ -955,6 +1018,8 @@ def choose_routes(self, veh_ids, route_choices): def get_x_by_id(self, veh_id): """See parent class.""" + if isinstance(veh_id, (list, np.ndarray)): + return [self.get_x_by_id(vehID) for vehID in veh_id] if self.get_edge(veh_id) == '': # occurs when a vehicle crashes is teleported for some other reason return 0. @@ -971,8 +1036,11 @@ def update_vehicle_colors(self): """ for veh_id in self.get_rl_ids(): try: - # color rl vehicles red - self.set_color(veh_id=veh_id, color=RED) + # If vehicle is already being colored via argument to vehicles.add(), don't re-color it. + if self._force_color_update or 'color' not in \ + self.type_parameters[self.get_type(veh_id)]: + # color rl vehicles red + self.set_color(veh_id=veh_id, color=RED) except (FatalTraCIError, TraCIException) as e: print('Error when updating rl vehicle colors:', e) @@ -980,10 +1048,36 @@ def update_vehicle_colors(self): for veh_id in self.get_human_ids(): try: color = CYAN if veh_id in self.get_observed_ids() else WHITE - self.set_color(veh_id=veh_id, color=color) + # If vehicle is already being colored via argument to vehicles.add(), don't re-color it. + if self._force_color_update or 'color' not in \ + self.type_parameters[self.get_type(veh_id)]: + self.set_color(veh_id=veh_id, color=color) except (FatalTraCIError, TraCIException) as e: print('Error when updating human vehicle colors:', e) + for veh_id in self.get_ids(): + try: + if 'av' in veh_id: + color = RED + # If vehicle is already being colored via argument to vehicles.add(), don't re-color it. + if self._force_color_update or 'color' not in \ + self.type_parameters[self.get_type(veh_id)]: + self.set_color(veh_id=veh_id, color=color) + except (FatalTraCIError, TraCIException) as e: + print('Error when updating human vehicle colors:', e) + + # color vehicles by speed if desired + if self._color_by_speed: + max_speed = self.master_kernel.network.max_speed() + speed_ranges = np.linspace(0, max_speed, STEPS) + for veh_id in self.get_ids(): + veh_speed = self.get_speed(veh_id) + bin_index = np.digitize(veh_speed, speed_ranges) + # If vehicle is already being colored via argument to vehicles.add(), don't re-color it. + if self._force_color_update or 'color' not in \ + self.type_parameters[self.get_type(veh_id)]: + self.set_color(veh_id=veh_id, color=color_bins[bin_index]) + # clear the list of observed vehicles for veh_id in self.get_observed_ids(): self.remove_observed(veh_id) @@ -1001,10 +1095,9 @@ def set_color(self, veh_id, color): The last term for sumo (transparency) is set to 255. """ - if self._color_vehicles: - r, g, b = color - self.kernel_api.vehicle.setColor( - vehID=veh_id, color=(r, g, b, 255)) + r, g, b = color + self.kernel_api.vehicle.setColor( + vehID=veh_id, color=(r, g, b, 255)) def add(self, veh_id, type_id, edge, pos, lane, speed): """See parent class.""" @@ -1035,3 +1128,52 @@ def get_max_speed(self, veh_id, error=-1001): def set_max_speed(self, veh_id, max_speed): """See parent class.""" self.kernel_api.vehicle.setMaxSpeed(veh_id, max_speed) + + def get_accel(self, veh_id, noise=True, failsafe=True): + """See parent class.""" + metric_name = 'accel' + if noise: + metric_name += '_with_noise' + else: + metric_name += '_no_noise' + if failsafe: + metric_name += '_with_falsafe' + else: + metric_name += '_no_failsafe' + + if metric_name not in self.__vehicles[veh_id]: + self.__vehicles[veh_id][metric_name] = None + return self.__vehicles[veh_id][metric_name] + + def update_accel(self, veh_id, accel, noise=True, failsafe=True): + """See parent class.""" + metric_name = 'accel' + if noise: + metric_name += '_with_noise' + else: + metric_name += '_no_noise' + if failsafe: + metric_name += '_with_falsafe' + else: + metric_name += '_no_failsafe' + + self.__vehicles[veh_id][metric_name] = accel + + def get_realized_accel(self, veh_id): + """See parent class.""" + if self.get_distance(veh_id) == 0: + return 0 + return (self.get_speed(veh_id) - self.get_previous_speed(veh_id)) / self.sim_step + + def get_2d_position(self, veh_id, error=-1001): + """See parent class.""" + return self.__sumo_obs.get(veh_id, {}).get(tc.VAR_POSITION, error) + + def get_distance(self, veh_id, error=-1001): + """See parent class.""" + return self.__sumo_obs.get(veh_id, {}).get(tc.VAR_DISTANCE, error) + + def get_road_grade(self, veh_id): + """See parent class.""" + # TODO : Brent + return 0 diff --git a/flow/core/params.py b/flow/core/params.py index e503186b1..79ad8d689 100755 --- a/flow/core/params.py +++ b/flow/core/params.py @@ -17,7 +17,27 @@ "all_checks": 31 } -LC_MODES = {"aggressive": 0, "no_lat_collide": 512, "strategic": 1621} +LC_MODES = { + "no_lc_safe": 512, + "no_lc_aggressive": 0, + "sumo_default": 1621, + "no_strategic_aggressive": 1108, + "no_strategic_safe": 1620, + "only_strategic_aggressive": 1, + "only_strategic_safe": 513, + "no_cooperative_aggressive": 1105, + "no_cooperative_safe": 1617, + "only_cooperative_aggressive": 4, + "only_cooperative_safe": 516, + "no_speed_gain_aggressive": 1093, + "no_speed_gain_safe": 1605, + "only_speed_gain_aggressive": 16, + "only_speed_gain_safe": 528, + "no_right_drive_aggressive": 1045, + "no_right_drive_safe": 1557, + "only_right_drive_aggressive": 64, + "only_right_drive_safe": 576 +} # Traffic light defaults PROGRAM_ID = 1 @@ -241,7 +261,8 @@ def add(self, initial_speed=0, num_vehicles=0, car_following_params=None, - lane_change_params=None): + lane_change_params=None, + color=None): """Add a sequence of vehicles to the list of vehicles in the network. Parameters @@ -292,6 +313,10 @@ def add(self, "car_following_params": car_following_params, "lane_change_params": lane_change_params} + if color: + type_params['color'] = color + self.type_parameters[veh_id]['color'] = color + # TODO: delete? self.initial.append({ "veh_id": @@ -384,7 +409,7 @@ class SimParams(object): specifies whether to render the radius of RL observation pxpm : int, optional specifies rendering resolution (pixel / meter) - color_vehicles : bool, optional + force_color_update : bool, optional whether or not to automatically color vehicles according to their types """ @@ -397,7 +422,7 @@ def __init__(self, sight_radius=25, show_radius=False, pxpm=2, - color_vehicles=True): + force_color_update=False): """Instantiate SimParams.""" self.sim_step = sim_step self.render = render @@ -407,7 +432,7 @@ def __init__(self, self.sight_radius = sight_radius self.pxpm = pxpm self.show_radius = show_radius - self.color_vehicles = color_vehicles + self.force_color_update = force_color_update class AimsunParams(SimParams): @@ -445,7 +470,7 @@ class AimsunParams(SimParams): specifies whether to render the radius of RL observation pxpm : int, optional specifies rendering resolution (pixel / meter) - scenario_name : str, optional + network_name : str, optional name of the network generated in Aimsun. experiment_name : str, optional name of the experiment generated in Aimsun @@ -453,7 +478,7 @@ class AimsunParams(SimParams): name of the replication generated in Aimsun. When loading an Aimsun template, this parameter must be set to the name of the replication to be run by the simulation; in this case, - the scenario_name and experiment_name parameters are not + the network_name and experiment_name parameters are not necessary as they will be obtained from the replication name. centroid_config_name : str, optional name of the centroid configuration to load in Aimsun. This @@ -477,7 +502,7 @@ def __init__(self, show_radius=False, pxpm=2, # set to match Flow_Aimsun.ang's scenario name - scenario_name="Dynamic Scenario 866", + network_name="Dynamic Scenario 866", # set to match Flow_Aimsun.ang's experiment name experiment_name="Micro SRC Experiment 867", # set to match Flow_Aimsun.ang's replication name @@ -488,7 +513,7 @@ def __init__(self, super(AimsunParams, self).__init__( sim_step, render, restart_instance, emission_path, save_render, sight_radius, show_radius, pxpm) - self.scenario_name = scenario_name + self.network_name = network_name self.experiment_name = experiment_name self.replication_name = replication_name self.centroid_config_name = centroid_config_name @@ -539,6 +564,8 @@ class SumoParams(SimParams): specifies whether to render the radius of RL observation pxpm : int, optional specifies rendering resolution (pixel / meter) + force_color_update : bool, optional + whether or not to automatically color vehicles according to their types overtake_right : bool, optional whether vehicles are allowed to overtake on the right as well as the left @@ -556,6 +583,11 @@ class SumoParams(SimParams): they teleport after teleport_time seconds num_clients : int, optional Number of clients that will connect to Traci + color_by_speed : bool + whether to color the vehicles by the speed they are moving at the + current time step + use_ballistic: bool, optional + If true, use a ballistic integration step instead of an euler step """ def __init__(self, @@ -569,16 +601,19 @@ def __init__(self, sight_radius=25, show_radius=False, pxpm=2, + force_color_update=False, overtake_right=False, seed=None, restart_instance=False, print_warnings=True, teleport_time=-1, - num_clients=1): + num_clients=1, + color_by_speed=False, + use_ballistic=False): """Instantiate SumoParams.""" super(SumoParams, self).__init__( sim_step, render, restart_instance, emission_path, save_render, - sight_radius, show_radius, pxpm) + sight_radius, show_radius, pxpm, force_color_update) self.port = port self.lateral_resolution = lateral_resolution self.no_step_log = no_step_log @@ -587,6 +622,8 @@ def __init__(self, self.print_warnings = print_warnings self.teleport_time = teleport_time self.num_clients = num_clients + self.color_by_speed = color_by_speed + self.use_ballistic = use_ballistic class EnvParams: @@ -880,14 +917,71 @@ class SumoLaneChangeParams: ---------- lane_change_mode : str or int, optional may be one of the following: + * "no_lc_safe" (default): Disable all SUMO lane changing but still + handle safety checks (collision avoidance and safety-gap enforcement) + in the simulation. Binary is [001000000000] + * "no_lc_aggressive": SUMO lane changes are not executed, collision + avoidance and safety-gap enforcement are off. + Binary is [000000000000] + + * "sumo_default": Execute all changes requested by a custom controller + unless in conflict with TraCI. Binary is [011001010101]. + + * "no_strategic_aggressive": Execute all changes except strategic + (routing) lane changes unless in conflict with TraCI. Collision + avoidance and safety-gap enforcement are off. Binary is [010001010100] + * "no_strategic_safe": Execute all changes except strategic + (routing) lane changes unless in conflict with TraCI. Collision + avoidance and safety-gap enforcement are on. Binary is [011001010100] + * "only_strategic_aggressive": Execute only strategic (routing) lane + changes unless in conflict with TraCI. Collision avoidance and + safety-gap enforcement are off. Binary is [000000000001] + * "only_strategic_safe": Execute only strategic (routing) lane + changes unless in conflict with TraCI. Collision avoidance and + safety-gap enforcement are on. Binary is [001000000001] + + * "no_cooperative_aggressive": Execute all changes except cooperative + (change in order to allow others to change) lane changes unless in + conflict with TraCI. Collision avoidance and safety-gap enforcement + are off. Binary is [010001010001] + * "no_cooperative_safe": Execute all changes except cooperative + lane changes unless in conflict with TraCI. Collision avoidance and + safety-gap enforcement are on. Binary is [011001010001] + * "only_cooperative_aggressive": Execute only cooperative lane changes + unless in conflict with TraCI. Collision avoidance and safety-gap + enforcement are off. Binary is [000000000100] + * "only_cooperative_safe": Execute only cooperative lane changes + unless in conflict with TraCI. Collision avoidance and safety-gap + enforcement are on. Binary is [001000000100] + + * "no_speed_gain_aggressive": Execute all changes except speed gain (the + other lane allows for faster driving) lane changes unless in conflict + with TraCI. Collision avoidance and safety-gap enforcement are off. + Binary is [010001000101] + * "no_speed_gain_safe": Execute all changes except speed gain + lane changes unless in conflict with TraCI. Collision avoidance and + safety-gap enforcement are on. Binary is [011001000101] + * "only_speed_gain_aggressive": Execute only speed gain lane changes + unless in conflict with TraCI. Collision avoidance and safety-gap + enforcement are off. Binary is [000000010000] + * "only_speed_gain_safe": Execute only speed gain lane changes + unless in conflict with TraCI. Collision avoidance and safety-gap + enforcement are on. Binary is [001000010000] + + * "no_right_drive_aggressive": Execute all changes except right drive + (obligation to drive on the right) lane changes unless in conflict + with TraCI. Collision avoidance and safety-gap enforcement are off. + Binary is [010000010101] + * "no_right_drive_safe": Execute all changes except right drive + lane changes unless in conflict with TraCI. Collision avoidance and + safety-gap enforcement are on. Binary is [011000010101] + * "only_right_drive_aggressive": Execute only right drive lane changes + unless in conflict with TraCI. Collision avoidance and safety-gap + enforcement are off. Binary is [000001000000] + * "only_right_drive_safe": Execute only right drive lane changes + unless in conflict with TraCI. Collision avoidance and safety-gap + enforcement are on. Binary is [001001000000] - * "no_lat_collide" (default): Human cars will not make lane - changes, RL cars can lane change into any space, no matter how - likely it is to crash - * "strategic": Human cars make lane changes in accordance with SUMO - to provide speed boosts - * "aggressive": RL cars are not limited by sumo with regard to - their lane-change actions, and can crash longitudinally * int values may be used to define custom lane change modes for the given vehicles, specified at: http://sumo.dlr.de/wiki/TraCI/Change_Vehicle_State#lane_change_mode_.280xb6.29 @@ -914,10 +1008,6 @@ class SumoLaneChangeParams: see lcPushyGap in Note lc_assertive : float, optional see lcAssertive in Note - lc_impatience : float, optional - see lcImpatience in Note - lc_time_to_impatience : float, optional - see lcTimeToImpatience in Note lc_accel_lat : float, optional see lcAccelLate in Note kwargs : dict @@ -930,7 +1020,7 @@ class SumoLaneChangeParams: """ def __init__(self, - lane_change_mode="no_lat_collide", + lane_change_mode="no_lc_safe", model="LC2013", lc_strategic=1.0, lc_cooperative=1.0, @@ -942,8 +1032,6 @@ def __init__(self, lc_pushy=0, lc_pushy_gap=0.6, lc_assertive=1, - lc_impatience=0, - lc_time_to_impatience=float("inf"), lc_accel_lat=1.0, **kwargs): """Instantiate SumoLaneChangeParams.""" @@ -998,17 +1086,6 @@ def __init__(self, deprecated_attribute(self, "lcAssertive", "lc_assertive") lc_assertive = kwargs["lcAssertive"] - # check for deprecations (lcImpatience) - if "lcImpatience" in kwargs: - deprecated_attribute(self, "lcImpatience", "lc_impatience") - lc_impatience = kwargs["lcImpatience"] - - # check for deprecations (lcTimeToImpatience) - if "lcTimeToImpatience" in kwargs: - deprecated_attribute(self, "lcTimeToImpatience", - "lc_time_to_impatience") - lc_time_to_impatience = kwargs["lcTimeToImpatience"] - # check for deprecations (lcAccelLat) if "lcAccelLat" in kwargs: deprecated_attribute(self, "lcAccelLat", "lc_accel_lat") @@ -1026,7 +1103,7 @@ def __init__(self, "lcCooperative": str(lc_cooperative), "lcSpeedGain": str(lc_speed_gain), "lcKeepRight": str(lc_keep_right), - # "lcLookaheadLeft": str(lcLookaheadLeft), + # "lcLookaheadLeft": str(lc_look_ahead_left), # "lcSpeedGainRight": str(lcSpeedGainRight) } elif model == "SL2015": @@ -1042,8 +1119,6 @@ def __init__(self, "lcPushy": str(lc_pushy), "lcPushyGap": str(lc_pushy_gap), "lcAssertive": str(lc_assertive), - "lcImpatience": str(lc_impatience), - "lcTimeToImpatience": str(lc_time_to_impatience), "lcAccelLat": str(lc_accel_lat) } @@ -1053,7 +1128,7 @@ def __init__(self, elif not (isinstance(lane_change_mode, int) or isinstance(lane_change_mode, float)): logging.error("Setting lane change mode to default.") - lane_change_mode = LC_MODES["no_lat_collide"] + lane_change_mode = LC_MODES["no_lc_safe"] self.lane_change_mode = lane_change_mode diff --git a/flow/core/rewards.py b/flow/core/rewards.py index 76c09b244..3cca916f5 100755 --- a/flow/core/rewards.py +++ b/flow/core/rewards.py @@ -237,7 +237,7 @@ def penalize_near_standstill(env, thresh=0.3, gain=1): This reward function is used to penalize vehicles below a specified threshold. This assists with discouraging RL from - gamifying a scenario, which can result in standstill behavior + gamifying a network, which can result in standstill behavior or similarly bad, near-zero velocities. Parameters @@ -304,3 +304,135 @@ def punish_rl_lane_changes(env, penalty=1): total_lane_change_penalty -= penalty return total_lane_change_penalty + + +def energy_consumption(env, gain=.001): + """Calculate power consumption of a vehicle. + + Assumes vehicle is an average sized vehicle. + The power calculated here is the lower bound of the actual power consumed + by a vehicle. + """ + power = 0 + + M = 1200 # mass of average sized vehicle (kg) + g = 9.81 # gravitational acceleration (m/s^2) + Cr = 0.005 # rolling resistance coefficient + Ca = 0.3 # aerodynamic drag coefficient + rho = 1.225 # air density (kg/m^3) + A = 2.6 # vehicle cross sectional area (m^2) + for veh_id in env.k.vehicle.get_ids(): + speed = env.k.vehicle.get_speed(veh_id) + prev_speed = env.k.vehicle.get_previous_speed(veh_id) + + accel = abs(speed - prev_speed) / env.sim_step + + power += M * speed * accel + M * g * Cr * speed + 0.5 * rho * A * Ca * speed ** 3 + + return -gain * power + + +def veh_energy_consumption(env, veh_id, gain=.001): + """Calculate power consumption of a vehicle. + + Assumes vehicle is an average sized vehicle. + The power calculated here is the lower bound of the actual power consumed + by a vehicle. + """ + power = 0 + + M = 1200 # mass of average sized vehicle (kg) + g = 9.81 # gravitational acceleration (m/s^2) + Cr = 0.005 # rolling resistance coefficient + Ca = 0.3 # aerodynamic drag coefficient + rho = 1.225 # air density (kg/m^3) + A = 2.6 # vehicle cross sectional area (m^2) + speed = env.k.vehicle.get_speed(veh_id) + prev_speed = env.k.vehicle.get_previous_speed(veh_id) + + accel = abs(speed - prev_speed) / env.sim_step + + power += M * speed * accel + M * g * Cr * speed + 0.5 * rho * A * Ca * speed ** 3 + + return -gain * power + + +def miles_per_megajoule(env, veh_ids=None, gain=.001): + """Calculate miles per mega-joule of either a particular vehicle or the total average of all the vehicles. + + Assumes vehicle is an average sized vehicle. + The power calculated here is the lower bound of the actual power consumed + by a vehicle. + + Parameters + ---------- + env : flow.envs.Env + the environment variable, which contains information on the current + state of the system. + veh_ids : [list] + list of veh_ids to compute the reward over + gain : float + scaling factor for the reward + """ + mpj = 0 + counter = 0 + if veh_ids is None: + veh_ids = env.k.vehicle.get_ids() + elif not isinstance(veh_ids, list): + veh_ids = [veh_ids] + for veh_id in veh_ids: + speed = env.k.vehicle.get_speed(veh_id) + # convert to be positive since the function called is a penalty + power = -veh_energy_consumption(env, veh_id, gain=1.0) + if power > 0 and speed >= 0.0: + counter += 1 + # meters / joule is (v * \delta t) / (power * \delta t) + mpj += speed / power + if counter > 0: + mpj /= counter + + # convert from meters per joule to miles per joule + mpj /= 1609.0 + # convert from miles per joule to miles per megajoule + mpj *= 10**6 + + return mpj * gain + + +def miles_per_gallon(env, veh_ids=None, gain=.001): + """Calculate mpg of either a particular vehicle or the total average of all the vehicles. + + Assumes vehicle is an average sized vehicle. + The power calculated here is the lower bound of the actual power consumed + by a vehicle. + + Parameters + ---------- + env : flow.envs.Env + the environment variable, which contains information on the current + state of the system. + veh_ids : [list] + list of veh_ids to compute the reward over + gain : float + scaling factor for the reward + """ + mpg = 0 + counter = 0 + if veh_ids is None: + veh_ids = env.k.vehicle.get_ids() + elif not isinstance(veh_ids, list): + veh_ids = [veh_ids] + for veh_id in veh_ids: + speed = env.k.vehicle.get_speed(veh_id) + gallons_per_s = env.k.vehicle.get_fuel_consumption(veh_id) + if gallons_per_s > 0 and speed >= 0.0: + counter += 1 + # meters / gallon is (v * \delta t) / (gallons_per_s * \delta t) + mpg += speed / gallons_per_s + if counter > 0: + mpg /= counter + + # convert from meters per gallon to miles per gallon + mpg /= 1609.0 + + return mpg * gain diff --git a/flow/envs/__init__.py b/flow/envs/__init__.py index 5befe6a33..611ed3d9a 100755 --- a/flow/envs/__init__.py +++ b/flow/envs/__init__.py @@ -4,7 +4,7 @@ from flow.envs.bottleneck import BottleneckAccelEnv, BottleneckEnv, \ BottleneckDesiredVelocityEnv from flow.envs.traffic_light_grid import TrafficLightGridEnv, \ - TrafficLightGridPOEnv, TrafficLightGridTestEnv + TrafficLightGridPOEnv, TrafficLightGridTestEnv, TrafficLightGridBenchmarkEnv from flow.envs.ring.lane_change_accel import LaneChangeAccelEnv, \ LaneChangeAccelPOEnv from flow.envs.ring.accel import AccelEnv @@ -33,6 +33,7 @@ 'WaveAttenuationPOEnv', 'TrafficLightGridEnv', 'TrafficLightGridPOEnv', + 'TrafficLightGridBenchmarkEnv', 'BottleneckDesiredVelocityEnv', 'TestEnv', 'BayBridgeEnv', diff --git a/flow/envs/base.py b/flow/envs/base.py old mode 100755 new mode 100644 index 0bc9324ae..c4462e8c8 --- a/flow/envs/base.py +++ b/flow/envs/base.py @@ -1,5 +1,6 @@ """Base environment class. This is the parent of all other environments.""" +from abc import ABCMeta, abstractmethod from copy import deepcopy import os import atexit @@ -7,6 +8,8 @@ import traceback import numpy as np import random +import shutil +import subprocess from flow.renderer.pyglet_renderer import PygletRenderer as Renderer from flow.utils.flow_warnings import deprecated_attribute @@ -24,7 +27,7 @@ from flow.utils.exceptions import FatalFlowError -class Env(gym.Env): +class Env(gym.Env, metaclass=ABCMeta): """Base environment class. Provides the interface for interacting with various aspects of a traffic @@ -127,7 +130,10 @@ def __init__(self, self.network = scenario if scenario is not None else network self.net_params = self.network.net_params self.initial_config = self.network.initial_config - self.sim_params = sim_params + self.sim_params = deepcopy(sim_params) + # check whether we should be rendering + self.should_render = self.sim_params.render + self.sim_params.render = False time_stamp = ''.join(str(time.time()).split('.')) if os.environ.get("TEST_FLAG", 0): # 1.0 works with stress_test_start 10k times @@ -151,7 +157,7 @@ def __init__(self, # create the Flow kernel self.k = Kernel(simulator=self.simulator, - sim_params=sim_params) + sim_params=self.sim_params) # use the network class's network parameters to generate the necessary # network components within the network kernel @@ -164,7 +170,7 @@ def __init__(self, # the network kernel as an input in order to determine what network # needs to be simulated. kernel_api = self.k.simulation.start_simulation( - network=self.k.network, sim_params=sim_params) + network=self.k.network, sim_params=self.sim_params) # pass the kernel api to the kernel and it's subclasses self.k.pass_api(kernel_api) @@ -214,7 +220,10 @@ def __init__(self, # render a frame self.render(reset=True) elif self.sim_params.render in [True, False]: - pass # default to sumo-gui (if True) or sumo (if False) + # default to sumo-gui (if True) or sumo (if False) + if (self.sim_params.render is True) and self.sim_params.save_render: + self.path = os.path.expanduser('~')+'/flow_rendering/' + self.network.name + os.makedirs(self.path, exist_ok=True) else: raise FatalFlowError( 'Mode %s is not supported!' % self.sim_params.render) @@ -387,8 +396,9 @@ def step(self, rl_actions): # test if the environment should terminate due to a collision or the # time horizon being met - done = (self.time_counter >= self.env_params.warmup_steps + - self.env_params.horizon) # or crash + done = (self.time_counter >= self.env_params.sims_per_step * + (self.env_params.warmup_steps + self.env_params.horizon) + or crash) # compute the info for each agent infos = {} @@ -421,6 +431,13 @@ def reset(self): # reset the time counter self.time_counter = 0 + # Now that we've passed the possibly fake init steps some rl libraries + # do, we can feel free to actually render things + if self.should_render: + self.sim_params.render = True + # got to restart the simulation to make it actually display anything + self.restart_simulation(self.sim_params) + # warn about not using restart_instance when using inflows if len(self.net_params.inflows.get()) > 0 and \ not self.sim_params.restart_instance: @@ -471,6 +488,9 @@ def reset(self): except (FatalTraCIError, TraCIException): print("Error during start: {}".format(traceback.format_exc())) + # do any additional resetting of the vehicle class needed + self.k.vehicle.reset() + # reintroduce the initial vehicles to the network for veh_id in self.initial_ids: type_id, edge, lane_index, pos, speed = \ @@ -595,9 +615,11 @@ def apply_rl_actions(self, rl_actions=None): rl_clipped = self.clip_actions(rl_actions) self._apply_rl_actions(rl_clipped) + @abstractmethod def _apply_rl_actions(self, rl_actions): - raise NotImplementedError + pass + @abstractmethod def get_state(self): """Return the state of the simulation as perceived by the RL agent. @@ -609,9 +631,10 @@ def get_state(self): information on the state of the vehicles, which is provided to the agent """ - raise NotImplementedError + pass @property + @abstractmethod def action_space(self): """Identify the dimensions and bounds of the action space. @@ -622,9 +645,10 @@ def action_space(self): gym Box or Tuple type a bounded box depicting the shape and bounds of the action space """ - raise NotImplementedError + pass @property + @abstractmethod def observation_space(self): """Identify the dimensions and bounds of the observation space. @@ -636,7 +660,7 @@ def observation_space(self): a bounded box depicting the shape and bounds of the observation space """ - raise NotImplementedError + pass def compute_reward(self, rl_actions, **kwargs): """Reward function for the RL agent(s). @@ -670,6 +694,15 @@ def terminate(self): # close pyglet renderer if self.sim_params.render in ['gray', 'dgray', 'rgb', 'drgb']: self.renderer.close() + # generate video + elif (self.sim_params.render is True) and self.sim_params.save_render: + images_dir = self.path.split('/')[-1] + speedup = 10 # multiplier: renders video so that `speedup` seconds is rendered in 1 real second + fps = speedup//self.sim_step + p = subprocess.Popen(["ffmpeg", "-y", "-r", str(fps), "-i", self.path+"/frame_%06d.png", + "-pix_fmt", "yuv420p", "%s/../%s.mp4" % (self.path, images_dir)]) + p.wait() + shutil.rmtree(self.path) except FileNotFoundError: # Skip automatic termination. Connection is probably already closed print(traceback.format_exc()) @@ -699,6 +732,9 @@ def render(self, reset=False, buffer_length=5): if len(self.frame_buffer) > buffer_length: self.frame_buffer.pop(0) self.sights_buffer.pop(0) + elif (self.sim_params.render is True) and self.sim_params.save_render: + # sumo-gui render + self.k.kernel_api.gui.screenshot("View #0", self.path+"/frame_%06d.png" % self.time_counter) def pyglet_render(self): """Render a frame using pyglet.""" diff --git a/flow/envs/bay_bridge.py b/flow/envs/bay_bridge.py index dcc5e991d..0c5570367 100644 --- a/flow/envs/bay_bridge.py +++ b/flow/envs/bay_bridge.py @@ -234,6 +234,22 @@ def compute_reward(self, rl_actions, **kwargs): # The below methods need to be updated by child classes. # ########################################################################### + @property + def action_space(self): + """See parent class. + + To be implemented by child classes. + """ + pass + + @property + def observation_space(self): + """See parent class. + + To be implemented by child classes. + """ + pass + def _apply_rl_actions(self, rl_actions): """See parent class. diff --git a/flow/envs/bottleneck.py b/flow/envs/bottleneck.py index c647e1d6e..a5f1508f9 100644 --- a/flow/envs/bottleneck.py +++ b/flow/envs/bottleneck.py @@ -472,6 +472,13 @@ def observation_space(self): shape=(1, ), dtype=np.float32) + def _apply_rl_actions(self, rl_actions): + """See parent class. + + To be implemented by child classes. + """ + pass + def compute_reward(self, rl_actions, **kwargs): """Outflow rate over last ten seconds normalized to max of 1.""" reward = self.k.vehicle.get_outflow_rate(10 * self.sim_step) / \ diff --git a/flow/envs/multiagent/__init__.py b/flow/envs/multiagent/__init__.py index 4c43611aa..f7889591d 100644 --- a/flow/envs/multiagent/__init__.py +++ b/flow/envs/multiagent/__init__.py @@ -3,10 +3,23 @@ from flow.envs.multiagent.base import MultiEnv from flow.envs.multiagent.ring.wave_attenuation import \ MultiWaveAttenuationPOEnv - -from flow.envs.multiagent.ring.accel import MultiAgentAccelEnv +from flow.envs.multiagent.ring.wave_attenuation import \ + MultiAgentWaveAttenuationPOEnv +from flow.envs.multiagent.ring.accel import AdversarialAccelEnv +from flow.envs.multiagent.ring.accel import MultiAgentAccelPOEnv from flow.envs.multiagent.traffic_light_grid import MultiTrafficLightGridPOEnv from flow.envs.multiagent.highway import MultiAgentHighwayPOEnv +from flow.envs.multiagent.merge import MultiAgentMergePOEnv +from flow.envs.multiagent.i210 import I210MultiEnv -__all__ = ['MultiEnv', 'MultiAgentAccelEnv', 'MultiWaveAttenuationPOEnv', - 'MultiTrafficLightGridPOEnv', 'MultiAgentHighwayPOEnv'] +__all__ = [ + 'MultiEnv', + 'AdversarialAccelEnv', + 'MultiWaveAttenuationPOEnv', + 'MultiTrafficLightGridPOEnv', + 'MultiAgentHighwayPOEnv', + 'MultiAgentAccelPOEnv', + 'MultiAgentWaveAttenuationPOEnv', + 'MultiAgentMergePOEnv', + 'I210MultiEnv' +] diff --git a/flow/envs/multiagent/base.py b/flow/envs/multiagent/base.py index 4753c9f45..2d9c3cd78 100644 --- a/flow/envs/multiagent/base.py +++ b/flow/envs/multiagent/base.py @@ -108,7 +108,8 @@ def step(self, rl_actions): states = self.get_state() done = {key: key in self.k.vehicle.get_arrived_ids() for key in states.keys()} - if crash: + if crash or (self.time_counter >= self.env_params.sims_per_step * + (self.env_params.warmup_steps + self.env_params.horizon)): done['__all__'] = True else: done['__all__'] = False @@ -121,6 +122,11 @@ def step(self, rl_actions): else: reward = self.compute_reward(rl_actions, fail=crash) + for rl_id in self.k.vehicle.get_arrived_rl_ids(self.env_params.sims_per_step): + done[rl_id] = True + reward[rl_id] = 0 + states[rl_id] = np.zeros(self.observation_space.shape[0]) + return states, reward, done, infos def reset(self, new_inflow_rate=None): @@ -142,6 +148,13 @@ def reset(self, new_inflow_rate=None): # reset the time counter self.time_counter = 0 + # Now that we've passed the possibly fake init steps some rl libraries + # do, we can feel free to actually render things + if self.should_render: + self.sim_params.render = True + # got to restart the simulation to make it actually display anything + self.restart_simulation(self.sim_params) + # warn about not using restart_instance when using inflows if len(self.net_params.inflows.get()) > 0 and \ not self.sim_params.restart_instance: @@ -192,6 +205,9 @@ def reset(self, new_inflow_rate=None): except (FatalTraCIError, TraCIException): print("Error during start: {}".format(traceback.format_exc())) + # do any additional resetting of the vehicle class needed + self.k.vehicle.reset() + # reintroduce the initial vehicles to the network for veh_id in self.initial_ids: type_id, edge, lane_index, pos, speed = \ diff --git a/flow/envs/multiagent/i210.py b/flow/envs/multiagent/i210.py new file mode 100644 index 000000000..409aeb14f --- /dev/null +++ b/flow/envs/multiagent/i210.py @@ -0,0 +1,225 @@ +"""Environment for training vehicles to reduce congestion in the I210.""" + +from gym.spaces import Box +import numpy as np + +from flow.core.rewards import average_velocity +from flow.envs.multiagent.base import MultiEnv + +# largest number of lanes on any given edge in the network +MAX_LANES = 6 + +ADDITIONAL_ENV_PARAMS = { + # maximum acceleration for autonomous vehicles, in m/s^2 + "max_accel": 1, + # maximum deceleration for autonomous vehicles, in m/s^2 + "max_decel": 1, + # whether we use an obs space that contains adjacent lane info or just the lead obs + "lead_obs": True, +} + + +class I210MultiEnv(MultiEnv): + """Partially observable multi-agent environment for the I-210 subnetworks. + + The policy is shared among the agents, so there can be a non-constant + number of RL vehicles throughout the simulation. + + Required from env_params: + + * max_accel: maximum acceleration for autonomous vehicles, in m/s^2 + * max_decel: maximum deceleration for autonomous vehicles, in m/s^2 + + The following states, actions and rewards are considered for one autonomous + vehicle only, as they will be computed in the same way for each of them. + + States + The observation consists of the speeds and bumper-to-bumper headways of + the vehicles immediately preceding and following autonomous vehicles in + all of the preceding lanes as well, a binary value indicating which of + these vehicles is autonomous, and the speed of the autonomous vehicle. + Missing vehicles are padded with zeros. + + Actions + The action consists of an acceleration, bound according to the + environment parameters, as well as three values that will be converted + into probabilities via softmax to decide of a lane change (left, none + or right). NOTE: lane changing is currently not enabled. It's a TODO. + + Rewards + The reward function encourages proximity of the system-level velocity + to a desired velocity specified in the environment parameters, while + slightly penalizing small time headways among autonomous vehicles. + + Termination + A rollout is terminated if the time horizon is reached or if two + vehicles collide into one another. + """ + + def __init__(self, env_params, sim_params, network, simulator='traci'): + super().__init__(env_params, sim_params, network, simulator) + self.lead_obs = env_params.additional_params.get("lead_obs") + + @property + def observation_space(self): + """See class definition.""" + # speed, speed of leader, headway + if self.lead_obs: + return Box( + low=-float('inf'), + high=float('inf'), + shape=(3,), + dtype=np.float32 + ) + # speed, dist to ego vehicle, binary value which is 1 if the vehicle is + # an AV + else: + leading_obs = 3 * MAX_LANES + follow_obs = 3 * MAX_LANES + + # speed and lane + self_obs = 2 + + return Box( + low=-float('inf'), + high=float('inf'), + shape=(leading_obs + follow_obs + self_obs,), + dtype=np.float32 + ) + + @property + def action_space(self): + """See class definition.""" + return Box( + low=-np.abs(self.env_params.additional_params['max_decel']), + high=self.env_params.additional_params['max_accel'], + shape=(1,), # (4,), + dtype=np.float32) + + def _apply_rl_actions(self, rl_actions): + """See class definition.""" + # in the warmup steps, rl_actions is None + if rl_actions: + for rl_id, actions in rl_actions.items(): + accel = actions[0] + + # lane_change_softmax = np.exp(actions[1:4]) + # lane_change_softmax /= np.sum(lane_change_softmax) + # lane_change_action = np.random.choice([-1, 0, 1], + # p=lane_change_softmax) + + self.k.vehicle.apply_acceleration(rl_id, accel) + # self.k.vehicle.apply_lane_change(rl_id, lane_change_action) + + def get_state(self): + """See class definition.""" + if self.lead_obs: + veh_info = {} + for rl_id in self.k.vehicle.get_rl_ids(): + speed = self.k.vehicle.get_speed(rl_id) + headway = self.k.vehicle.get_headway(rl_id) + lead_speed = self.k.vehicle.get_speed(self.k.vehicle.get_leader(rl_id)) + if lead_speed == -1001: + lead_speed = 0 + veh_info.update({rl_id: np.array([speed / 50.0, headway / 1000.0, lead_speed / 50.0])}) + else: + veh_info = {rl_id: np.concatenate((self.state_util(rl_id), + self.veh_statistics(rl_id))) + for rl_id in self.k.vehicle.get_rl_ids()} + return veh_info + + def compute_reward(self, rl_actions, **kwargs): + # TODO(@evinitsky) we need something way better than this. Something that adds + # in notions of local reward + """See class definition.""" + # in the warmup steps + if rl_actions is None: + return {} + + rewards = {} + for rl_id in self.k.vehicle.get_rl_ids(): + if self.env_params.evaluate: + # reward is speed of vehicle if we are in evaluation mode + reward = self.k.vehicle.get_speed(rl_id) + elif kwargs['fail']: + # reward is 0 if a collision occurred + reward = 0 + else: + # reward high system-level velocities + cost1 = average_velocity(self, fail=kwargs['fail']) + + # penalize small time headways + cost2 = 0 + t_min = 1 # smallest acceptable time headway + + lead_id = self.k.vehicle.get_leader(rl_id) + if lead_id not in ["", None] \ + and self.k.vehicle.get_speed(rl_id) > 0: + t_headway = max( + self.k.vehicle.get_headway(rl_id) / + self.k.vehicle.get_speed(rl_id), 0) + cost2 += min((t_headway - t_min) / t_min, 0) + + # weights for cost1, cost2, and cost3, respectively + eta1, eta2 = 1.00, 0.10 + + reward = max(eta1 * cost1 + eta2 * cost2, 0) + + rewards[rl_id] = reward + return rewards + + def additional_command(self): + """See parent class. + + Define which vehicles are observed for visualization purposes. + """ + # specify observed vehicles + for rl_id in self.k.vehicle.get_rl_ids(): + # leader + lead_id = self.k.vehicle.get_leader(rl_id) + if lead_id: + self.k.vehicle.set_observed(lead_id) + # follower + follow_id = self.k.vehicle.get_follower(rl_id) + if follow_id: + self.k.vehicle.set_observed(follow_id) + + def state_util(self, rl_id): + """Return an array of headway, tailway, leader speed, follower speed. + + Also return a 1 if leader is rl 0 otherwise, a 1 if follower is rl 0 otherwise. + If there are fewer than MAX_LANES the extra + entries are filled with -1 to disambiguate from zeros. + """ + veh = self.k.vehicle + lane_headways = veh.get_lane_headways(rl_id).copy() + lane_tailways = veh.get_lane_tailways(rl_id).copy() + lane_leader_speed = veh.get_lane_leaders_speed(rl_id).copy() + lane_follower_speed = veh.get_lane_followers_speed(rl_id).copy() + leader_ids = veh.get_lane_leaders(rl_id).copy() + follower_ids = veh.get_lane_followers(rl_id).copy() + rl_ids = self.k.vehicle.get_rl_ids() + is_leader_rl = [1 if l_id in rl_ids else 0 for l_id in leader_ids] + is_follow_rl = [1 if f_id in rl_ids else 0 for f_id in follower_ids] + diff = MAX_LANES - len(is_leader_rl) + if diff > 0: + # the minus 1 disambiguates missing cars from missing lanes + lane_headways += diff * [-1] + lane_tailways += diff * [-1] + lane_leader_speed += diff * [-1] + lane_follower_speed += diff * [-1] + is_leader_rl += diff * [-1] + is_follow_rl += diff * [-1] + lane_headways = np.asarray(lane_headways) / 1000 + lane_tailways = np.asarray(lane_tailways) / 1000 + lane_leader_speed = np.asarray(lane_leader_speed) / 100 + lane_follower_speed = np.asarray(lane_follower_speed) / 100 + return np.concatenate((lane_headways, lane_tailways, lane_leader_speed, + lane_follower_speed, is_leader_rl, + is_follow_rl)) + + def veh_statistics(self, rl_id): + """Return speed, edge information, and x, y about the vehicle itself.""" + speed = self.k.vehicle.get_speed(rl_id) / 100.0 + lane = (self.k.vehicle.get_lane(rl_id) + 1) / 10.0 + return np.array([speed, lane]) diff --git a/flow/envs/multiagent/merge.py b/flow/envs/multiagent/merge.py new file mode 100644 index 000000000..42189419a --- /dev/null +++ b/flow/envs/multiagent/merge.py @@ -0,0 +1,190 @@ +"""Environment for training vehicles to reduce congestion in a merge.""" + +from flow.envs.multiagent.base import MultiEnv +from flow.core import rewards +from gym.spaces.box import Box +import numpy as np + + +ADDITIONAL_ENV_PARAMS = { + # maximum acceleration for autonomous vehicles, in m/s^2 + "max_accel": 3, + # maximum deceleration for autonomous vehicles, in m/s^2 + "max_decel": 3, + # desired velocity for all vehicles in the network, in m/s + "target_velocity": 25, +} + + +class MultiAgentMergePOEnv(MultiEnv): + """Partially observable multi-agent merge environment. + + This environment is used to train autonomous vehicles to attenuate the + formation and propagation of waves in an open merge network. + + Required from env_params: + + * max_accel: maximum acceleration for autonomous vehicles, in m/s^2 + * max_decel: maximum deceleration for autonomous vehicles, in m/s^2 + * target_velocity: desired velocity for all vehicles in the network, in m/s + + States + The observation consists of the speeds and bumper-to-bumper headways of + the vehicles immediately preceding and following autonomous vehicle, as + well as the ego speed of the autonomous vehicles. + + In order to maintain a fixed observation size, when the number of AVs + in the network is less than "num_rl", the extra entries are filled in + with zeros. Conversely, if the number of autonomous vehicles is greater + than "num_rl", the observations from the additional vehicles are not + included in the state space. + + Actions + The action space consists of a vector of bounded accelerations for each + autonomous vehicle $i$. In order to ensure safety, these actions are + bounded by failsafes provided by the simulator at every time step. + + In order to account for variability in the number of autonomous + vehicles, if n_AV < "num_rl" the additional actions provided by the + agent are not assigned to any vehicle. Moreover, if n_AV > "num_rl", + the additional vehicles are not provided with actions from the learning + agent, and instead act as human-driven vehicles as well. + + Rewards + The reward function encourages proximity of the system-level velocity + to a desired velocity, while slightly penalizing small time headways + among autonomous vehicles. + + Termination + A rollout is terminated if the time horizon is reached or if two + vehicles collide into one another. + """ + + def __init__(self, env_params, sim_params, network, simulator='traci'): + for p in ADDITIONAL_ENV_PARAMS.keys(): + if p not in env_params.additional_params: + raise KeyError( + 'Environment parameter "{}" not supplied'.format(p)) + + # used for visualization: the vehicles behind and after RL vehicles + # (ie the observed vehicles) will have a different color + self.leader = [] + self.follower = [] + + super().__init__(env_params, sim_params, network, simulator) + + @property + def action_space(self): + """See class definition.""" + return Box( + low=-abs(self.env_params.additional_params["max_decel"]), + high=self.env_params.additional_params["max_accel"], + shape=(1,), + dtype=np.float32) + + @property + def observation_space(self): + """See class definition.""" + return Box(low=-5, high=5, shape=(5,), dtype=np.float32) + + def _apply_rl_actions(self, rl_actions): + """See class definition.""" + for rl_id in enumerate(self.k.vehicle.get_rl_ids()): + if rl_id not in rl_actions.keys(): + # the vehicle just entered, so ignore + continue + self.k.vehicle.apply_acceleration(rl_id, rl_actions[rl_id]) + + def get_state(self, rl_id=None, **kwargs): + """See class definition.""" + observation = {} + self.leader = [] + self.follower = [] + + # normalizing constants + max_speed = self.k.network.max_speed() + max_length = self.k.network.length() + + for rl_id in self.k.vehicle.get_rl_ids(): + this_speed = self.k.vehicle.get_speed(rl_id) + lead_id = self.k.vehicle.get_leader(rl_id) + follower = self.k.vehicle.get_follower(rl_id) + + if lead_id in ["", None]: + # in case leader is not visible + lead_speed = max_speed + lead_head = max_length + else: + self.leader.append(lead_id) + lead_speed = self.k.vehicle.get_speed(lead_id) + lead_head = self.k.vehicle.get_x_by_id(lead_id) \ + - self.k.vehicle.get_x_by_id(rl_id) \ + - self.k.vehicle.get_length(rl_id) + + if follower in ["", None]: + # in case follower is not visible + follow_speed = 0 + follow_head = max_length + else: + self.follower.append(follower) + follow_speed = self.k.vehicle.get_speed(follower) + follow_head = self.k.vehicle.get_headway(follower) + + observation[rl_id] = np.array([ + this_speed / max_speed, + (lead_speed - this_speed) / max_speed, + lead_head / max_length, + (this_speed - follow_speed) / max_speed, + follow_head / max_length + ]) + + return observation + + def compute_reward(self, rl_actions, **kwargs): + """See class definition.""" + if self.env_params.evaluate: + return np.mean(self.k.vehicle.get_speed(self.k.vehicle.get_ids())) + else: + # return a reward of 0 if a collision occurred + if kwargs["fail"]: + return 0 + + # reward high system-level velocities + cost1 = rewards.desired_velocity(self, fail=kwargs["fail"]) + + # penalize small time headways + cost2 = 0 + t_min = 1 # smallest acceptable time headway + for rl_id in self.k.vehicle.get_rl_ids(): + lead_id = self.k.vehicle.get_leader(rl_id) + if lead_id not in ["", None] \ + and self.k.vehicle.get_speed(rl_id) > 0: + t_headway = max( + self.k.vehicle.get_headway(rl_id) / + self.k.vehicle.get_speed(rl_id), 0) + cost2 += min((t_headway - t_min) / t_min, 0) + + # weights for cost1 and cost2, respectively + eta1, eta2 = 1.00, 0.10 + + reward = max(eta1 * cost1 + eta2 * cost2, 0) + return {key: reward for key in self.k.vehicle.get_rl_ids()} + + def additional_command(self): + """See parent class. + + This method defines which vehicles are observed for visualization + purposes. + """ + for veh_id in self.leader + self.follower: + self.k.vehicle.set_observed(veh_id) + + def reset(self, new_inflow_rate=None): + """See parent class. + + In addition, a few variables that are specific to this class are + emptied before they are used by the new rollout. + """ + self.leader = [] + self.follower = [] + return super().reset() diff --git a/flow/envs/multiagent/ring/accel.py b/flow/envs/multiagent/ring/accel.py index 9db207b1b..cffd0948e 100644 --- a/flow/envs/multiagent/ring/accel.py +++ b/flow/envs/multiagent/ring/accel.py @@ -1,16 +1,49 @@ """Environment for training the acceleration behavior of vehicles in a ring.""" - import numpy as np +from gym.spaces import Box + from flow.core import rewards from flow.envs.ring.accel import AccelEnv from flow.envs.multiagent.base import MultiEnv -class MultiAgentAccelEnv(AccelEnv, MultiEnv): - """Adversarial multi-agent env. +ADDITIONAL_ENV_PARAMS = { + # maximum acceleration for autonomous vehicles, in m/s^2 + "max_accel": 1, + # maximum deceleration for autonomous vehicles, in m/s^2 + "max_decel": 1, + # desired velocity for all vehicles in the network, in m/s + "target_velocity": 20, +} + + +class AdversarialAccelEnv(AccelEnv, MultiEnv): + """Adversarial multi-agent acceleration env. + + States + The observation of both the AV and adversary agent consist of the + velocities and absolute position of all vehicles in the network. This + assumes a constant number of vehicles. + + Actions + * AV: The action space of the AV agent consists of a vector of bounded + accelerations for each autonomous vehicle. In order to ensure safety, + these actions are further bounded by failsafes provided by the + simulator at every time step. + * Adversary: The action space of the adversary agent consists of a + vector of perturbations to the accelerations issued by the AV agent. + These are directly added to the original accelerations by the AV + agent. - Multi-agent env with an adversarial agent perturbing - the accelerations of the autonomous vehicle + Rewards + * AV: The reward for the AV agent is equal to the mean speed of all + vehicles in the network. + * Adversary: The adversary receives a reward equal to the negative + reward issued to the AV agent. + + Termination + A rollout is terminated if the time horizon is reached or if two + vehicles collide into one another. """ def _apply_rl_actions(self, rl_actions): @@ -50,3 +83,147 @@ def get_state(self, **kwargs): ] for veh_id in self.sorted_ids]) state = np.ndarray.flatten(state) return {'av': state, 'adversary': state} + + +class MultiAgentAccelPOEnv(MultiEnv): + """Multi-agent acceleration environment for shared policies. + + This environment can used to train autonomous vehicles to achieve certain + desired speeds in a decentralized fashion. This should be applicable to + both closed and open network settings. + + Required from env_params: + + * max_accel: maximum acceleration for autonomous vehicles, in m/s^2 + * max_decel: maximum deceleration for autonomous vehicles, in m/s^2 + * target_velocity: desired velocity for all vehicles in the network, in m/s + + States + The observation of each agent (i.e. each autonomous vehicle) consists + of the speeds and bumper-to-bumper headways of the vehicles immediately + preceding and following autonomous vehicle, as well as the absolute + position and ego speed of the autonomous vehicles. This results in a + state space of size 6 for each agent. + + Actions + The action space for each agent consists of a scalar bounded + acceleration for each autonomous vehicle. In order to ensure safety, + these actions are further bounded by failsafes provided by the + simulator at every time step. + + Rewards + The reward function is the two-norm of the distance of the speed of the + vehicles in the network from the "target_velocity" term. For a + description of the reward, see: flow.core.rewards.desired_speed. This + reward is shared by all agents. + + Termination + A rollout is terminated if the time horizon is reached or if two + vehicles collide into one another. + """ + + def __init__(self, env_params, sim_params, network, simulator='traci'): + for p in ADDITIONAL_ENV_PARAMS.keys(): + if p not in env_params.additional_params: + raise KeyError( + 'Environment parameter "{}" not supplied'.format(p)) + + self.leader = [] + self.follower = [] + + super().__init__(env_params, sim_params, network, simulator) + + @property + def action_space(self): + """See class definition.""" + return Box( + low=-abs(self.env_params.additional_params["max_decel"]), + high=self.env_params.additional_params["max_accel"], + shape=(1, ), + dtype=np.float32) + + @property + def observation_space(self): + """See class definition.""" + return Box(low=-5, high=5, shape=(6,), dtype=np.float32) + + def _apply_rl_actions(self, rl_actions): + """See class definition.""" + for veh_id in self.k.vehicle.get_rl_ids(): + self.k.vehicle.apply_acceleration(veh_id, rl_actions[veh_id]) + + def compute_reward(self, rl_actions, **kwargs): + """See class definition.""" + # Compute the common reward. + reward = rewards.desired_velocity(self, fail=kwargs['fail']) + + # Reward is shared by all agents. + return {key: reward for key in self.k.vehicle.get_rl_ids()} + + def get_state(self, **kwargs): # FIXME + """See class definition.""" + self.leader = [] + self.follower = [] + obs = {} + + # normalizing constants + max_speed = self.k.network.max_speed() + max_length = self.k.network.length() + + for rl_id in self.k.vehicle.get_rl_ids(): + this_pos = self.k.vehicle.get_x_by_id(rl_id) + this_speed = self.k.vehicle.get_speed(rl_id) + lead_id = self.k.vehicle.get_leader(rl_id) + follower = self.k.vehicle.get_follower(rl_id) + + if lead_id in ["", None]: + # in case leader is not visible + lead_speed = max_speed + lead_head = max_length + else: + self.leader.append(lead_id) + lead_speed = self.k.vehicle.get_speed(lead_id) + lead_head = self.k.vehicle.get_x_by_id(lead_id) \ + - self.k.vehicle.get_x_by_id(rl_id) \ + - self.k.vehicle.get_length(rl_id) + + if follower in ["", None]: + # in case follower is not visible + follow_speed = 0 + follow_head = max_length + else: + self.follower.append(follower) + follow_speed = self.k.vehicle.get_speed(follower) + follow_head = self.k.vehicle.get_headway(follower) + + # Add the next observation. + obs[rl_id] = np.array([ + this_pos / max_length, + this_speed / max_speed, + (lead_speed - this_speed) / max_speed, + lead_head / max_length, + (this_speed - follow_speed) / max_speed, + follow_head / max_length + ]) + + return obs + + def additional_command(self): + """See parent class. + + This method defines which vehicles are observed for visualization + purposes. + """ + # specify observed vehicles + for veh_id in self.leader + self.follower: + self.k.vehicle.set_observed(veh_id) + + def reset(self): + """See parent class. + + In addition, a few variables that are specific to this class are + emptied before they are used by the new rollout. + """ + self.leader = [] + self.follower = [] + return super().reset() diff --git a/flow/envs/multiagent/ring/wave_attenuation.py b/flow/envs/multiagent/ring/wave_attenuation.py index b464c6514..811616c0f 100644 --- a/flow/envs/multiagent/ring/wave_attenuation.py +++ b/flow/envs/multiagent/ring/wave_attenuation.py @@ -10,7 +10,15 @@ import numpy as np from gym.spaces.box import Box +import random +from scipy.optimize import fsolve +from copy import deepcopy + +from flow.core.params import InitialConfig +from flow.core.params import NetParams from flow.envs.multiagent.base import MultiEnv +from flow.envs.ring.wave_attenuation import v_eq_max_function + ADDITIONAL_ENV_PARAMS = { # maximum acceleration of autonomous vehicles @@ -35,6 +43,7 @@ class MultiWaveAttenuationPOEnv(MultiEnv): States See parent class + Actions See parent class @@ -43,13 +52,12 @@ class MultiWaveAttenuationPOEnv(MultiEnv): Termination See parent class - """ @property def observation_space(self): """See class definition.""" - return Box(low=0, high=1, shape=(3,), dtype=np.float32) + return Box(low=-1, high=1, shape=(3,), dtype=np.float32) @property def action_space(self): @@ -125,7 +133,180 @@ def additional_command(self): lead_id = self.k.vehicle.get_leader(rl_id) or rl_id self.k.vehicle.set_observed(lead_id) - def gen_edges(self, i): + @staticmethod + def gen_edges(i): """Return the edges corresponding to the rl id.""" return ['top_{}'.format(i), 'left_{}'.format(i), 'right_{}'.format(i), 'bottom_{}'.format(i)] + + +class MultiAgentWaveAttenuationPOEnv(MultiEnv): + """Multi-agent variant of WaveAttenuationPOEnv. + + Required from env_params: + + * max_accel: maximum acceleration of autonomous vehicles + * max_decel: maximum deceleration of autonomous vehicles + * ring_length: bounds on the ranges of ring road lengths the autonomous + vehicle is trained on. If set to None, the environment sticks to the ring + road specified in the original network definition. + + States + The state of each agent (AV) consists of the speed and headway of the + ego vehicle, as well as the difference in speed between the ego vehicle + and its leader. There is no assumption on the number of vehicles in the + network. + + Actions + Actions are an acceleration for each rl vehicle, bounded by the maximum + accelerations and decelerations specified in EnvParams. + + Rewards + The reward function rewards high average speeds from all vehicles in + the network, and penalizes accelerations by the rl vehicle. This reward + is shared by all agents. + + Termination + A rollout is terminated if the time horizon is reached or if two + vehicles collide into one another. + """ + + def __init__(self, env_params, sim_params, network, simulator='traci'): + for p in ADDITIONAL_ENV_PARAMS.keys(): + if p not in env_params.additional_params: + raise KeyError( + 'Environment parameter \'{}\' not supplied'.format(p)) + + super().__init__(env_params, sim_params, network, simulator) + + @property + def observation_space(self): + """See class definition.""" + return Box(low=-5, high=5, shape=(3,), dtype=np.float32) + + @property + def action_space(self): + """See class definition.""" + return Box( + low=-np.abs(self.env_params.additional_params['max_decel']), + high=self.env_params.additional_params['max_accel'], + shape=(1,), + dtype=np.float32) + + def get_state(self): + """See class definition.""" + obs = {} + for rl_id in self.k.vehicle.get_rl_ids(): + lead_id = self.k.vehicle.get_leader(rl_id) or rl_id + + # normalizers + max_speed = 15. + max_length = self.env_params.additional_params['ring_length'][1] + + observation = np.array([ + self.k.vehicle.get_speed(rl_id) / max_speed, + (self.k.vehicle.get_speed(lead_id) - + self.k.vehicle.get_speed(rl_id)) + / max_speed, + self.k.vehicle.get_headway(rl_id) / max_length + ]) + obs.update({rl_id: observation}) + + return obs + + def _apply_rl_actions(self, rl_actions): + """Split the accelerations by ring.""" + if rl_actions: + rl_ids = list(rl_actions.keys()) + accel = list(rl_actions.values()) + self.k.vehicle.apply_acceleration(rl_ids, accel) + + def compute_reward(self, rl_actions, **kwargs): + """See class definition.""" + # in the warmup steps + if rl_actions is None: + return 0 + + vel = np.array([ + self.k.vehicle.get_speed(veh_id) + for veh_id in self.k.vehicle.get_ids() + ]) + + if any(vel < -100) or kwargs['fail']: + return 0. + + # reward average velocity + eta_2 = 4. + reward = eta_2 * np.mean(vel) / 20 + + # punish accelerations (should lead to reduced stop-and-go waves) + eta = 4 # 0.25 + mean_actions = np.mean(np.abs(list(rl_actions.values()))) + accel_threshold = 0 + + if mean_actions > accel_threshold: + reward += eta * (accel_threshold - mean_actions) + + return {key: reward for key in self.k.vehicle.get_rl_ids()} + + def additional_command(self): + """Define which vehicles are observed for visualization purposes.""" + # specify observed vehicles + for rl_id in self.k.vehicle.get_rl_ids(): + lead_id = self.k.vehicle.get_leader(rl_id) or rl_id + self.k.vehicle.set_observed(lead_id) + + def reset(self, new_inflow_rate=None): + """See parent class. + + The sumo instance is reset with a new ring length, and a number of + steps are performed with the rl vehicle acting as a human vehicle. + """ + # skip if ring length is None + if self.env_params.additional_params['ring_length'] is None: + return super().reset() + + # reset the step counter + self.step_counter = 0 + + # update the network + initial_config = InitialConfig(bunching=50, min_gap=0) + length = random.randint( + self.env_params.additional_params['ring_length'][0], + self.env_params.additional_params['ring_length'][1]) + additional_net_params = { + 'length': + length, + 'lanes': + self.net_params.additional_params['lanes'], + 'speed_limit': + self.net_params.additional_params['speed_limit'], + 'resolution': + self.net_params.additional_params['resolution'] + } + net_params = NetParams(additional_params=additional_net_params) + + self.network = self.network.__class__( + self.network.orig_name, self.network.vehicles, + net_params, initial_config) + self.k.vehicle = deepcopy(self.initial_vehicles) + self.k.vehicle.kernel_api = self.k.kernel_api + self.k.vehicle.master_kernel = self.k + + # solve for the velocity upper bound of the ring + v_guess = 4 + v_eq_max = fsolve(v_eq_max_function, np.array(v_guess), + args=(len(self.initial_ids), length))[0] + + print('\n-----------------------') + print('ring length:', net_params.additional_params['length']) + print('v_max:', v_eq_max) + print('-----------------------') + + # restart the sumo instance + self.restart_simulation( + sim_params=self.sim_params, + render=self.sim_params.render) + + # perform the generic reset function + return super().reset() diff --git a/flow/envs/multiagent/traffic_light_grid.py b/flow/envs/multiagent/traffic_light_grid.py index 66854dd1d..a0438f828 100644 --- a/flow/envs/multiagent/traffic_light_grid.py +++ b/flow/envs/multiagent/traffic_light_grid.py @@ -70,7 +70,7 @@ def observation_space(self): high=1, shape=(3 * 4 * self.num_observed + 2 * self.num_local_edges + - 3 * (1 + self.num_local_lights), + 2 * (1 + self.num_local_lights), ), dtype=np.float32) return tl_box @@ -167,14 +167,12 @@ def get_state(self): self.observed_ids = all_observed_ids # Traffic light information - last_change = self.last_change.flatten() direction = self.direction.flatten() currently_yellow = self.currently_yellow.flatten() # This is a catch-all for when the relative_node method returns a -1 # (when there is no node in the direction sought). We add a last # item to the lists here, which will serve as a default value. # TODO(cathywu) are these values reasonable? - last_change = np.append(last_change, [0]) direction = np.append(direction, [0]) currently_yellow = np.append(currently_yellow, [1]) @@ -194,7 +192,7 @@ def get_state(self): observation = np.array(np.concatenate( [speeds[rl_id_num], dist_to_intersec[rl_id_num], edge_number[rl_id_num], density[local_edge_numbers], - velocity_avg[local_edge_numbers], last_change[local_id_nums], + velocity_avg[local_edge_numbers], direction[local_id_nums], currently_yellow[local_id_nums] ])) obs.update({rl_id: observation}) diff --git a/flow/envs/ring/wave_attenuation.py b/flow/envs/ring/wave_attenuation.py index 3aeb037b1..ec10db16d 100644 --- a/flow/envs/ring/wave_attenuation.py +++ b/flow/envs/ring/wave_attenuation.py @@ -207,12 +207,7 @@ def reset(self): render=self.sim_params.render) # perform the generic reset function - observation = super().reset() - - # reset the timer to zero - self.time_counter = 0 - - return observation + return super().reset() class WaveAttenuationPOEnv(WaveAttenuationEnv): diff --git a/flow/envs/traffic_light_grid.py b/flow/envs/traffic_light_grid.py index 4083ac3fe..8be0cb8a5 100644 --- a/flow/envs/traffic_light_grid.py +++ b/flow/envs/traffic_light_grid.py @@ -639,7 +639,7 @@ def observation_space(self): """ tl_box = Box( low=0., - high=1, + high=3, shape=(3 * 4 * self.num_observed * self.num_traffic_lights + 2 * len(self.k.network.get_edge_list()) + 3 * self.num_traffic_lights,), @@ -731,6 +731,17 @@ def additional_command(self): [self.k.vehicle.set_observed(veh_id) for veh_id in self.observed_ids] +class TrafficLightGridBenchmarkEnv(TrafficLightGridPOEnv): + """Class used for the benchmarks in `Benchmarks for reinforcement learning inmixed-autonomy traffic`.""" + + def compute_reward(self, rl_actions, **kwargs): + """See class definition.""" + if self.env_params.evaluate: + return - rewards.min_delay_unscaled(self) + else: + return rewards.desired_velocity(self) + + class TrafficLightGridTestEnv(TrafficLightGridEnv): """ Class for use in testing. diff --git a/flow/multiagent_envs/__iniy__.py b/flow/multiagent_envs/__init__.py similarity index 83% rename from flow/multiagent_envs/__iniy__.py rename to flow/multiagent_envs/__init__.py index e270fa397..75ffa34dc 100644 --- a/flow/multiagent_envs/__iniy__.py +++ b/flow/multiagent_envs/__init__.py @@ -3,13 +3,13 @@ from flow.multiagent_envs.multiagent_env import MultiEnv from flow.multiagent_envs.loop.wave_attenuation import \ MultiWaveAttenuationPOEnv -from flow.multiagent_envs.loop.loop_accel import MultiAgentAccelEnv +from flow.multiagent_envs.loop.loop_accel import AdversarialAccelEnv from flow.multiagent_envs.traffic_light_grid import MultiTrafficLightGridPOEnv from flow.multiagent_envs.highway import MultiAgentHighwayPOEnv __all__ = [ 'MultiEnv', - 'MultiAgentAccelEnv', + 'AdversarialAccelEnv', 'MultiWaveAttenuationPOEnv', 'MultiTrafficLightGridPOEnv', 'MultiAgentHighwayPOEnv' diff --git a/flow/multiagent_envs/loop/loop_accel.py b/flow/multiagent_envs/loop/loop_accel.py index f46e3e38d..2fe79f421 100644 --- a/flow/multiagent_envs/loop/loop_accel.py +++ b/flow/multiagent_envs/loop/loop_accel.py @@ -1,14 +1,14 @@ """Pending deprecation file. -To view the actual content, go to: flow/envs/multiagent/traffic_light_grid.py +To view the actual content, go to: flow/envs/multiagent/ring/accel.py """ from flow.utils.flow_warnings import deprecated -from flow.envs.multiagent.ring.accel import MultiAgentAccelEnv as MAAEnv +from flow.envs.multiagent.ring.accel import AdversarialAccelEnv as MAAEnv @deprecated('flow.multiagent_envs.loop.loop_accel', - 'flow.envs.multiagent.ring.accel.MultiAgentAccelEnv') -class MultiAgentAccelEnv(MAAEnv): + 'flow.envs.multiagent.ring.accel.AdversarialAccelEnv') +class AdversarialAccelEnv(MAAEnv): """See parent class.""" pass diff --git a/flow/networks/__init__.py b/flow/networks/__init__.py index a3677d83a..af849031d 100644 --- a/flow/networks/__init__.py +++ b/flow/networks/__init__.py @@ -15,10 +15,11 @@ from flow.networks.multi_ring import MultiRingNetwork from flow.networks.minicity import MiniCityNetwork from flow.networks.highway_ramps import HighwayRampsNetwork +from flow.networks.i210_subnetwork import I210SubNetwork __all__ = [ "Network", "BayBridgeNetwork", "BayBridgeTollNetwork", "BottleneckNetwork", "FigureEightNetwork", "TrafficLightGridNetwork", "HighwayNetwork", "RingNetwork", "MergeNetwork", "MultiRingNetwork", - "MiniCityNetwork", "HighwayRampsNetwork" + "MiniCityNetwork", "HighwayRampsNetwork", "I210SubNetwork" ] diff --git a/flow/networks/base.py b/flow/networks/base.py index 2979a4b27..58964348f 100644 --- a/flow/networks/base.py +++ b/flow/networks/base.py @@ -1,4 +1,4 @@ -"""Contains the base scenario class.""" +"""Contains the base network class.""" from flow.core.params import InitialConfig from flow.core.params import TrafficLightParams diff --git a/flow/networks/highway.py b/flow/networks/highway.py index c63292067..e48331cf9 100644 --- a/flow/networks/highway.py +++ b/flow/networks/highway.py @@ -13,7 +13,14 @@ # speed limit for all edges "speed_limit": 30, # number of edges to divide the highway into - "num_edges": 1 + "num_edges": 1, + # whether to include a ghost edge. This edge is provided a different speed + # limit. + "use_ghost_edge": False, + # speed limit for the ghost edge + "ghost_speed_limit": 25, + # length of the downstream ghost edge with the reduced speed limit + "boundary_cell_length": 500 } @@ -29,6 +36,11 @@ class HighwayNetwork(Network): * **lanes** : number of lanes in the highway * **speed_limit** : max speed limit of the highway * **num_edges** : number of edges to divide the highway into + * **use_ghost_edge** : whether to include a ghost edge. This edge is + provided a different speed limit. + * **ghost_speed_limit** : speed limit for the ghost edge + * **boundary_cell_length** : length of the downstream ghost edge with the + reduced speed limit Usage ----- @@ -62,10 +74,6 @@ def __init__(self, if p not in net_params.additional_params: raise KeyError('Network parameter "{}" not supplied'.format(p)) - self.length = net_params.additional_params["length"] - self.lanes = net_params.additional_params["lanes"] - self.num_edges = net_params.additional_params.get("num_edges", 1) - super().__init__(name, vehicles, net_params, initial_config, traffic_lights) @@ -74,6 +82,7 @@ def specify_nodes(self, net_params): length = net_params.additional_params["length"] num_edges = net_params.additional_params.get("num_edges", 1) segment_lengths = np.linspace(0, length, num_edges+1) + end_length = net_params.additional_params["boundary_cell_length"] nodes = [] for i in range(num_edges+1): @@ -83,6 +92,13 @@ def specify_nodes(self, net_params): "y": 0 }] + if self.net_params.additional_params["use_ghost_edge"]: + nodes += [{ + "id": "edge_{}".format(num_edges + 1), + "x": length + end_length, + "y": 0 + }] + return nodes def specify_edges(self, net_params): @@ -90,6 +106,7 @@ def specify_edges(self, net_params): length = net_params.additional_params["length"] num_edges = net_params.additional_params.get("num_edges", 1) segment_length = length/float(num_edges) + end_length = net_params.additional_params["boundary_cell_length"] edges = [] for i in range(num_edges): @@ -101,12 +118,22 @@ def specify_edges(self, net_params): "length": segment_length }] + if self.net_params.additional_params["use_ghost_edge"]: + edges += [{ + "id": "highway_end", + "type": "highway_end", + "from": "edge_{}".format(num_edges), + "to": "edge_{}".format(num_edges + 1), + "length": end_length + }] + return edges def specify_types(self, net_params): """See parent class.""" lanes = net_params.additional_params["lanes"] speed_limit = net_params.additional_params["speed_limit"] + end_speed_limit = net_params.additional_params["ghost_speed_limit"] types = [{ "id": "highwayType", @@ -114,6 +141,13 @@ def specify_types(self, net_params): "speed": speed_limit }] + if self.net_params.additional_params["use_ghost_edge"]: + types += [{ + "id": "highway_end", + "numLanes": lanes, + "speed": end_speed_limit + }] + return types def specify_routes(self, net_params): @@ -123,14 +157,51 @@ def specify_routes(self, net_params): for i in range(num_edges): rts["highway_{}".format(i)] = ["highway_{}".format(j) for j in range(i, num_edges)] + if self.net_params.additional_params["use_ghost_edge"]: + rts["highway_{}".format(i)].append("highway_end") return rts def specify_edge_starts(self): """See parent class.""" - edgestarts = [("highway_{}".format(i), 0) - for i in range(self.num_edges)] - return edgestarts + junction_length = 0.1 + length = self.net_params.additional_params["length"] + num_edges = self.net_params.additional_params.get("num_edges", 1) + + # Add the main edges. + edge_starts = [ + ("highway_{}".format(i), + i * (length / num_edges + junction_length)) + for i in range(num_edges) + ] + + if self.net_params.additional_params["use_ghost_edge"]: + edge_starts += [ + ("highway_end", length + num_edges * junction_length) + ] + + return edge_starts + + def specify_internal_edge_starts(self): + """See parent class.""" + junction_length = 0.1 + length = self.net_params.additional_params["length"] + num_edges = self.net_params.additional_params.get("num_edges", 1) + + # Add the junctions. + edge_starts = [ + (":edge_{}".format(i + 1), + (i + 1) * length / num_edges + i * junction_length) + for i in range(num_edges - 1) + ] + + if self.net_params.additional_params["use_ghost_edge"]: + edge_starts += [ + (":edge_{}".format(num_edges), + length + (num_edges - 1) * junction_length) + ] + + return edge_starts @staticmethod def gen_custom_start_pos(cls, net_params, initial_config, num_vehicles): diff --git a/flow/networks/i210_subnetwork.py b/flow/networks/i210_subnetwork.py new file mode 100644 index 000000000..f4315b07f --- /dev/null +++ b/flow/networks/i210_subnetwork.py @@ -0,0 +1,286 @@ +"""Contains the I-210 sub-network class.""" +from flow.networks.base import Network +from flow.core.params import InitialConfig +from flow.core.params import TrafficLightParams + +ADDITIONAL_NET_PARAMS = { + # whether to include vehicle on the on-ramp + "on_ramp": False, + # whether to include the downstream slow-down edge in the network + "ghost_edge": False, +} + +EDGES_DISTRIBUTION = [ + # Main highway + "ghost0", + "119257914", + "119257908#0", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3", + + # On-ramp + "27414345", + "27414342#0", + "27414342#1-AddedOnRampEdge", + + # Off-ramp + "173381935", +] + + +class I210SubNetwork(Network): + """A network used to simulate the I-210 sub-network. + + Requires from net_params: + + * **on_ramp** : whether to include vehicle on the on-ramp + * **ghost_edge** : whether to include the downstream slow-down edge in the + network + + Usage + ----- + >>> from flow.core.params import NetParams + >>> from flow.core.params import VehicleParams + >>> from flow.core.params import InitialConfig + >>> from flow.networks import I210SubNetwork + >>> + >>> network = I210SubNetwork( + >>> name='I-210_subnetwork', + >>> vehicles=VehicleParams(), + >>> net_params=NetParams() + >>> ) + """ + + def __init__(self, + name, + vehicles, + net_params, + initial_config=InitialConfig(), + traffic_lights=TrafficLightParams()): + """Initialize the I210 sub-network scenario.""" + for p in ADDITIONAL_NET_PARAMS.keys(): + if p not in net_params.additional_params: + raise KeyError('Network parameter "{}" not supplied'.format(p)) + + # The length of each edge and junction is a fixed term that can be + # found in the xml file. + self.length_with_ghost_edge = [ + ("ghost0", 573.08), + (":300944378_0", 0.30), + ("119257914", 61.28), + (":300944379_0", 0.31), + ("119257908#0", 696.97), + (":300944436_0", 2.87), + ("119257908#1-AddedOnRampEdge", 97.20), + (":119257908#1-AddedOnRampNode_0", 3.24), + ("119257908#1", 239.68), + (":119257908#1-AddedOffRampNode_0", 3.24), + ("119257908#1-AddedOffRampEdge", 98.50), + (":1686591010_1", 5.46), + ("119257908#2", 576.61), + (":1842086610_1", 4.53), + ("119257908#3", 17.49), + ] + + super(I210SubNetwork, self).__init__( + name=name, + vehicles=vehicles, + net_params=net_params, + initial_config=initial_config, + traffic_lights=traffic_lights, + ) + + def specify_routes(self, net_params): + """See parent class.""" + rts = { + "119257914": [ + (["119257914", + "119257908#0", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ] + } + + if net_params.additional_params["ghost_edge"]: + rts.update({ + "ghost0": [ + (["ghost0", + "119257914", + "119257908#0", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1), + ], + }) + + if net_params.additional_params["on_ramp"]: + rts.update({ + # Main highway + "119257908#0": [ + (["119257908#0", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ], + "119257908#1-AddedOnRampEdge": [ + (["119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ], + "119257908#1": [ + (["119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ], + "119257908#1-AddedOffRampEdge": [ + (["119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1.0), + ], + "119257908#2": [ + (["119257908#2", + "119257908#3"], 1), + ], + "119257908#3": [ + (["119257908#3"], 1), + ], + + # On-ramp + "27414345": [ + (["27414345", + "27414342#1-AddedOnRampEdge", + "27414342#1", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1 - 9 / 321), + (["27414345", + "27414342#1-AddedOnRampEdge", + "27414342#1", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "173381935"], 9 / 321), + ], + "27414342#0": [ + (["27414342#0", + "27414342#1-AddedOnRampEdge", + "27414342#1", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 1 - 20 / 421), + (["27414342#0", + "27414342#1-AddedOnRampEdge", + "27414342#1", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "173381935"], 20 / 421), + ], + "27414342#1-AddedOnRampEdge": [ + (["27414342#1-AddedOnRampEdge", + "27414342#1", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "119257908#2", + "119257908#3"], 0.5), + (["27414342#1-AddedOnRampEdge", + "27414342#1", + "119257908#1-AddedOnRampEdge", + "119257908#1", + "119257908#1-AddedOffRampEdge", + "173381935"], 0.5), + ], + + # Off-ramp + "173381935": [ + (["173381935"], 1), + ], + }) + + return rts + + def specify_edge_starts(self): + """See parent class.""" + if self.net_params.additional_params["ghost_edge"]: + # Collect the names of all the edges. + edge_names = [ + e[0] for e in self.length_with_ghost_edge + if not e[0].startswith(":") + ] + + edge_starts = [] + for edge in edge_names: + # Find the position of the edge in the list of tuples. + edge_pos = next( + i for i in range(len(self.length_with_ghost_edge)) + if self.length_with_ghost_edge[i][0] == edge + ) + + # Sum of lengths until the edge is reached to compute the + # starting position of the edge. + edge_starts.append(( + edge, + sum(e[1] for e in self.length_with_ghost_edge[:edge_pos]) + )) + + elif self.net_params.additional_params["on_ramp"]: + # TODO: this will incorporated in the future, if needed. + edge_starts = [] + + else: + # TODO: this will incorporated in the future, if needed. + edge_starts = [] + + return edge_starts + + def specify_internal_edge_starts(self): + """See parent class.""" + if self.net_params.additional_params["ghost_edge"]: + # Collect the names of all the junctions. + edge_names = [ + e[0] for e in self.length_with_ghost_edge + if e[0].startswith(":") + ] + + edge_starts = [] + for edge in edge_names: + # Find the position of the edge in the list of tuples. + edge_pos = next( + i for i in range(len(self.length_with_ghost_edge)) + if self.length_with_ghost_edge[i][0] == edge + ) + + # Sum of lengths until the edge is reached to compute the + # starting position of the edge. + edge_starts.append(( + edge, + sum(e[1] for e in self.length_with_ghost_edge[:edge_pos]) + )) + + elif self.net_params.additional_params["on_ramp"]: + # TODO: this will incorporated in the future, if needed. + edge_starts = [] + + else: + # TODO: this will incorporated in the future, if needed. + edge_starts = [] + + return edge_starts diff --git a/flow/networks/ring.py b/flow/networks/ring.py index de4d17503..ceef22a78 100755 --- a/flow/networks/ring.py +++ b/flow/networks/ring.py @@ -37,7 +37,7 @@ class RingNetwork(Network): >>> from flow.core.params import NetParams >>> from flow.core.params import VehicleParams >>> from flow.core.params import InitialConfig - >>> from flow.scenarios import RingNetwork + >>> from flow.networks import RingNetwork >>> >>> network = RingNetwork( >>> name='ring_road', diff --git a/flow/scenarios/multi_loop.py b/flow/scenarios/multi_loop.py index 5a7b6d01e..e4fb666df 100644 --- a/flow/scenarios/multi_loop.py +++ b/flow/scenarios/multi_loop.py @@ -1,6 +1,6 @@ """Pending deprecation file. -To view the actual content, go to: flow/scenarios/multi_ring.py +To view the actual content, go to: flow/networks/multi_ring.py """ from flow.utils.flow_warnings import deprecated from flow.networks.multi_ring import MultiRingNetwork diff --git a/flow/utils/aimsun/generate.py b/flow/utils/aimsun/generate.py index 41590c96b..25ed213b0 100644 --- a/flow/utils/aimsun/generate.py +++ b/flow/utils/aimsun/generate.py @@ -29,6 +29,9 @@ "EPSG:32601") model = gui.getActiveModel() +# HACK: Store port in author +port_string = sys.argv[1] +model.setAuthor(port_string) def generate_net(nodes, edges, @@ -267,15 +270,15 @@ def generate_net(nodes, set_vehicles_color(model) # set API - scenario_name = data["scenario_name"] + network_name = data["network_name"] scenario = model.getCatalog().findByName( - scenario_name, model.getType("GKScenario")) # find scenario + network_name, model.getType("GKScenario")) # find scenario scenario_data = scenario.getInputData() scenario_data.addExtension(os.path.join( config.PROJECT_PATH, "flow/utils/aimsun/run.py"), True) # save - gui.saveAs('flow.ang') + gui.save(model, 'flow.ang', GGui.GGuiSaveType.eSaveAs) def generate_net_osm(file_name, inflows, veh_types): @@ -366,15 +369,15 @@ def generate_net_osm(file_name, inflows, veh_types): set_vehicles_color(model) # set API - scenario_name = data["scenario_name"] + network_name = data["network_name"] scenario = model.getCatalog().findByName( - scenario_name, model.getType("GKScenario")) # find scenario + network_name, model.getType("GKScenario")) # find scenario scenario_data = scenario.getInputData() scenario_data.addExtension(os.path.join( config.PROJECT_PATH, "flow/utils/aimsun/run.py"), True) # save - gui.saveAs('flow.ang') + gui.save(model, 'flow.ang', GGui.GGuiSaveType.eSaveAs) def get_junctions(nodes): @@ -685,7 +688,7 @@ def create_meter(model, edge): """ section = model.getCatalog().findByName(edge, model.getType("GKSection")) meter_length = 2 - pos = section.getLanesLength2D() - meter_length + pos = section.length2D() - meter_length type = model.getType("GKMetering") cmd = model.createNewCmd(model.getType("GKSectionObject")) # TODO double check the zeros @@ -795,7 +798,7 @@ def set_sim_step(experiment, sim_step): # collect the network-specific data -data_file = 'flow/core/kernel/network/data.json' +data_file = 'flow/core/kernel/network/data_%s.json'%port_string with open(os.path.join(config.PROJECT_PATH, data_file)) as f: data = json.load(f) @@ -825,13 +828,13 @@ def set_sim_step(experiment, sim_step): for s in types.itervalues(): s_id = s.getId() num_lanes = s.getNbFullLanes() - length = s.getLanesLength2D() + length = s.length2D() speed = s.getSpeed() edge_osm[s_id] = {"speed": speed, "length": length, "numLanes": num_lanes} with open(os.path.join(config.PROJECT_PATH, - 'flow/utils/aimsun/osm_edges.json'), 'w') \ + 'flow/utils/aimsun/osm_edges_%s.json' % port_string), 'w') \ as outfile: json.dump(edge_osm, outfile, sort_keys=True, indent=4) diff --git a/flow/utils/aimsun/load.py b/flow/utils/aimsun/load.py index fba3716f0..6310179c7 100644 --- a/flow/utils/aimsun/load.py +++ b/flow/utils/aimsun/load.py @@ -5,6 +5,7 @@ import flow.config as config from flow.utils.aimsun.scripting_api import AimsunTemplate +import sys def load_network(): @@ -56,21 +57,24 @@ def get_dict_from_objects(sections, nodes, turnings, cen_connections): if not centroid_config: print('[load.py] ERROR: Centroid configuration ' + centroid_config_name + ' does not exist.') - for c in centroid_config.origin_centroids: - scenario_data['centroids'][c.id] = {'type': 'in'} - for c in centroid_config.destination_centroids: - scenario_data['centroids'][c.id] = {'type': 'out'} + else: + # load origin centroids only if they exist + if centroid_config.origin_centroids: + for c in centroid_config.origin_centroids: + scenario_data['centroids'][c.id] = {'type': 'in'} + + # load destination centroids only if they exist + if centroid_config.destination_centroids: + for c in centroid_config.destination_centroids: + scenario_data['centroids'][c.id] = {'type': 'out'} # load sections for s in sections: scenario_data['sections'][s.id] = { 'name': s.name, 'numLanes': s.nb_full_lanes, - # FIXME this is a mean of the lanes lengths - # (bc they don't have to be all of the same size) - # it may not be 100% accurate - 'length': s.lanes_length_2D / s.nb_full_lanes, - 'max_speed': s.speed + 'length': s.length2D(), + 'speed': s.speed } # load nodes @@ -118,10 +122,11 @@ def get_dict_from_objects(sections, nodes, turnings, cen_connections): return scenario_data +port_string = sys.argv[1] # collect template path file_path = os.path.join(config.PROJECT_PATH, - 'flow/utils/aimsun/aimsun_template_path') + 'flow/utils/aimsun/aimsun_template_path_%s'%port_string) with open(file_path, 'r') as f: template_path = f.readline() os.remove(file_path) @@ -131,8 +136,11 @@ def get_dict_from_objects(sections, nodes, turnings, cen_connections): model = AimsunTemplate(GKSystem, GKGUISystem) model.load(template_path) +# HACK: Store port in author +model.setAuthor(port_string) + # collect the simulation parameters -params_file = 'flow/core/kernel/scenario/data.json' +params_file = 'flow/core/kernel/network/data_%s.json' % port_string params_path = os.path.join(config.PROJECT_PATH, params_file) with open(params_path) as f: data = json.load(f) @@ -168,7 +176,7 @@ def get_dict_from_objects(sections, nodes, turnings, cen_connections): scenario_data = load_network() # save template's scenario into a file to be loaded into Flow's scenario -scenario_data_file = 'flow/core/kernel/scenario/scenario_data.json' +scenario_data_file = 'flow/core/kernel/network/network_data_%s.json'%port_string scenario_data_path = os.path.join(config.PROJECT_PATH, scenario_data_file) with open(scenario_data_path, 'w') as f: json.dump(scenario_data, f, sort_keys=True, indent=4) @@ -177,7 +185,7 @@ def get_dict_from_objects(sections, nodes, turnings, cen_connections): # create a check file to announce that we are done # writing all the network data into the .json file -check_file = 'flow/core/kernel/scenario/scenario_data_check' +check_file = 'flow/core/kernel/network/network_data_check_%s'%port_string check_file_path = os.path.join(config.PROJECT_PATH, check_file) open(check_file_path, 'a').close() diff --git a/flow/utils/aimsun/run.py b/flow/utils/aimsun/run.py index adc9fa097..f290c8960 100644 --- a/flow/utils/aimsun/run.py +++ b/flow/utils/aimsun/run.py @@ -16,7 +16,8 @@ from thread import start_new_thread import numpy as np -PORT = 9999 +model = GKSystem.getSystem().getActiveModel() +PORT = int(model.getAuthor()) entered_vehicles = [] exited_vehicles = [] diff --git a/flow/utils/aimsun/small_template.ang b/flow/utils/aimsun/small_template.ang index f5d2ccd8a..645de560b 100644 Binary files a/flow/utils/aimsun/small_template.ang and b/flow/utils/aimsun/small_template.ang differ diff --git a/flow/utils/leaderboard/evaluate.py b/flow/utils/leaderboard/evaluate.py index fdac33c89..941309ca4 100644 --- a/flow/utils/leaderboard/evaluate.py +++ b/flow/utils/leaderboard/evaluate.py @@ -121,15 +121,56 @@ def get_state(self): env = env_class( env_params=env_params, sim_params=sim_params, network=network) - # create a Experiment object with the "rl_actions" method as - # described in the inputs. Note that the state may not be that which is + flow_params = dict( + # name of the experiment + exp_tag=exp_tag, + + # name of the flow environment the experiment is running on + env_name=env_class, + + # name of the network class the experiment is running on + network=network_class, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=sim_params, + + # environment related parameters (see flow.core.params.EnvParams) + env=env_params, + + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=net_params, + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=initial_config, + + # traffic lights to be introduced to specific nodes (see + # flow.core.params.TrafficLightParams) + tls=traffic_lights, + ) + + # number of time steps + flow_params['env'].horizon = env.env_params.horizon + + # create a Experiment object. Note that the state may not be that which is # specified by the environment. - exp = Experiment(env=env) + exp = Experiment(flow_params) + exp.env = env + + exp = Experiment(flow_params) + exp.env = env # run the experiment and return the reward res = exp.run( num_runs=NUM_RUNS, - num_steps=env.env_params.horizon, rl_actions=_get_actions) return np.mean(res["returns"]), np.std(res["returns"]) diff --git a/flow/utils/registry.py b/flow/utils/registry.py index 5670eb224..3f6c9dad5 100644 --- a/flow/utils/registry.py +++ b/flow/utils/registry.py @@ -63,9 +63,16 @@ def make_create_env(params, version=0, render=None): if isinstance(params["env_name"], str): print("""Passing of strings for env_name will be deprecated. Please pass the Env instance instead.""") - env_name = params["env_name"] + '-v{}'.format(version) + base_env_name = params["env_name"] else: - env_name = params["env_name"].__name__ + '-v{}'.format(version) + base_env_name = params["env_name"].__name__ + + # deal with multiple environments being created under the same name + all_envs = gym.envs.registry.all() + env_ids = [env_spec.id for env_spec in all_envs] + while "{}-v{}".format(base_env_name, version) in env_ids: + version += 1 + env_name = "{}-v{}".format(base_env_name, version) if isinstance(params["network"], str): print("""Passing of strings for network will be deprecated. @@ -109,18 +116,17 @@ def create_env(*_): else: entry_point = params["env_name"].__module__ + ':' + params["env_name"].__name__ - try: - register( - id=env_name, - entry_point=entry_point, - kwargs={ - "env_params": env_params, - "sim_params": sim_params, - "network": network, - "simulator": params['simulator'] - }) - except Exception: - pass + # register the environment with OpenAI gym + register( + id=env_name, + entry_point=entry_point, + kwargs={ + "env_params": env_params, + "sim_params": sim_params, + "network": network, + "simulator": params['simulator'] + }) + return gym.envs.make(env_name) return create_env, env_name diff --git a/flow/utils/rllib.py b/flow/utils/rllib.py index b5abc9a23..80193c22b 100644 --- a/flow/utils/rllib.py +++ b/flow/utils/rllib.py @@ -6,6 +6,7 @@ import json from copy import deepcopy import os +import sys import flow.envs from flow.core.params import SumoLaneChangeParams, SumoCarFollowingParams, \ @@ -207,6 +208,9 @@ def get_rllib_config(path): def get_rllib_pkl(path): """Return the data from the specified rllib configuration file.""" + dirname = os.path.dirname(__file__) + filename = os.path.join(dirname, '../../examples/') + sys.path.append(filename) config_path = os.path.join(path, "params.pkl") if not os.path.exists(config_path): config_path = os.path.join(path, "../params.pkl") diff --git a/flow/visualize/capacity_diagram_generator.py b/flow/visualize/capacity_diagram_generator.py index d4bc2e9a2..566d7e428 100644 --- a/flow/visualize/capacity_diagram_generator.py +++ b/flow/visualize/capacity_diagram_generator.py @@ -1,7 +1,7 @@ """Generates capacity diagrams for the bottleneck. This method accepts as input a csv file containing the inflows and outflows -from several simulations as created by the file `examples/sumo/density_exp.py`, +from several simulations as created by the file `examples/exp_scripts/bottleneck_density_sweep_capacity_diagram.py`, e.g. 1000, 978 diff --git a/flow/visualize/time_space_diagram.py b/flow/visualize/time_space_diagram.py index e712ed84b..4914fc6a7 100644 --- a/flow/visualize/time_space_diagram.py +++ b/flow/visualize/time_space_diagram.py @@ -17,61 +17,73 @@ python time_space_diagram.py .csv .json """ from flow.utils.rllib import get_flow_params -from flow.networks import RingNetwork, FigureEightNetwork, MergeNetwork -import csv -from matplotlib import pyplot as plt -from matplotlib.collections import LineCollection +from flow.networks import RingNetwork, FigureEightNetwork, MergeNetwork, I210SubNetwork, HighwayNetwork + +import argparse +from collections import defaultdict +try: + from matplotlib import pyplot as plt +except ImportError: + import matplotlib + matplotlib.use('TkAgg') + from matplotlib import pyplot as plt +from matplotlib.collections import LineCollection, PatchCollection +from matplotlib.patches import Rectangle import matplotlib.colors as colors import numpy as np -import argparse +import pandas as pd + # networks that can be plotted by this method ACCEPTABLE_NETWORKS = [ RingNetwork, FigureEightNetwork, MergeNetwork, + I210SubNetwork, + HighwayNetwork ] -def import_data_from_emission(fp): - r"""Import relevant data from the predefined emission (.csv) file. +def import_data_from_trajectory(fp, params=dict()): + r"""Import and preprocess data from the Flow trajectory (.csv) file. Parameters ---------- fp : str file path (for the .csv formatted file) + params : dict + flow-specific parameters, including: + + * "network" (str): name of the network that was used when generating + the emission file. Must be one of the network names mentioned in + ACCEPTABLE_NETWORKS, + * "net_params" (flow.core.params.NetParams): network-specific + parameters. This is used to collect the lengths of various network + links. Returns ------- - dict of dict - Key = "veh_id": name of the vehicle \n Elements: - - * "time": time step at every sample - * "edge": edge ID at every sample - * "pos": relative position at every sample - * "vel": speed at every sample + pd.DataFrame """ - # initialize all output variables - veh_id, t, edge, rel_pos, vel = [], [], [], [], [] - - # import relevant data from emission file - for record in csv.DictReader(open(fp)): - veh_id.append(record['id']) - t.append(record['time']) - edge.append(record['edge_id']) - rel_pos.append(record['relative_position']) - vel.append(record['speed']) - - # we now want to separate data by vehicle ID - ret = {key: {'time': [], 'edge': [], 'pos': [], 'vel': []} - for key in np.unique(veh_id)} - for i in range(len(veh_id)): - ret[veh_id[i]]['time'].append(float(t[i])) - ret[veh_id[i]]['edge'].append(edge[i]) - ret[veh_id[i]]['pos'].append(float(rel_pos[i])) - ret[veh_id[i]]['vel'].append(float(vel[i])) + # Read trajectory csv into pandas dataframe + df = pd.read_csv(fp) - return ret + # Convert column names for backwards compatibility using emissions csv + column_conversions = { + 'time': 'time_step', + 'lane_number': 'lane_id', + } + df = df.rename(columns=column_conversions) + if 'distance' not in df.columns: + df['distance'] = _get_abs_pos(df, params) + + # Compute line segment ends by shifting dataframe by 1 row + df[['next_pos', 'next_time']] = df.groupby('id')[['distance', 'time_step']].shift(-1) + + # Remove nans from data + df = df[df['next_time'].notna()] + + return df def get_time_space_data(data, params): @@ -79,13 +91,8 @@ def get_time_space_data(data, params): Parameters ---------- - data : dict of dict - Key = "veh_id": name of the vehicle \n Elements: - - * "time": time step at every sample - * "edge": edge ID at every sample - * "pos": relative position at every sample - * "vel": speed at every sample + data : pd.DataFrame + cleaned dataframe of the trajectory data params : dict flow-specific parameters, including: @@ -98,17 +105,14 @@ def get_time_space_data(data, params): Returns ------- - as_array - n_steps x n_veh matrix specifying the absolute position of every - vehicle at every time step. Set to zero if the vehicle is not present - in the network at that time step. - as_array - n_steps x n_veh matrix specifying the speed of every vehicle at every - time step. Set to zero if the vehicle is not present in the network at - that time step. - as_array - a (n_steps,) vector representing the unique time steps in the - simulation + ndarray (or dict < str, np.ndarray >) + 3d array (n_segments x 2 x 2) containing segments to be plotted. + every inner 2d array is comprised of two 1d arrays representing + [start time, start distance] and [end time, end distance] pairs. + + in the case of I210, the nested arrays are wrapped into a dict, + keyed on the lane number, so that each lane can be plotted + separately. Raises ------ @@ -117,172 +121,137 @@ def get_time_space_data(data, params): """ # check that the network is appropriate assert params['network'] in ACCEPTABLE_NETWORKS, \ - 'Network must be one of: ' + ', '.join(ACCEPTABLE_NETWORKS) + 'Network must be one of: ' + ', '.join([network.__name__ for network in ACCEPTABLE_NETWORKS]) # switcher used to compute the positions based on the type of network switcher = { RingNetwork: _ring_road, MergeNetwork: _merge, - FigureEightNetwork: _figure_eight + FigureEightNetwork: _figure_eight, + I210SubNetwork: _i210_subnetwork, + HighwayNetwork: _highway, } - # Collect a list of all the unique times. - all_time = [] - for veh_id in data.keys(): - all_time.extend(data[veh_id]['time']) - all_time = np.sort(np.unique(all_time)) - # Get the function from switcher dictionary func = switcher[params['network']] # Execute the function - pos, speed = func(data, params, all_time) + segs, data = func(data) - return pos, speed, all_time + return segs, data -def _merge(data, params, all_time): - r"""Generate position and speed data for the merge. +def _merge(data): + r"""Generate time and position data for the merge. This only include vehicles on the main highway, and not on the adjacent on-ramp. Parameters ---------- - data : dict of dict - Key = "veh_id": name of the vehicle \n Elements: + data : pd.DataFrame + cleaned dataframe of the trajectory data + + Returns + ------- + ndarray + 3d array (n_segments x 2 x 2) containing segments to be plotted. + every inner 2d array is comprised of two 1d arrays representing + [start time, start distance] and [end time, end distance] pairs. + pd.DataFrame + modified trajectory dataframe + """ + # Omit ghost edges + keep_edges = {'inflow_merge', 'bottom', ':bottom_0'} + data = data[data['edge_id'].isin(keep_edges)] + + segs = data[['time_step', 'distance', 'next_time', 'next_pos']].values.reshape((len(data), 2, 2)) + + return segs, data - * "time": time step at every sample - * "edge": edge ID at every sample - * "pos": relative position at every sample - * "vel": speed at every sample - params : dict - flow-specific parameters - all_time : array_like - a (n_steps,) vector representing the unique time steps in the - simulation + +def _highway(data): + r"""Generate time and position data for the highway. + + Parameters + ---------- + data : pd.DataFrame + cleaned dataframe of the trajectory data Returns ------- - as_array - n_steps x n_veh matrix specifying the absolute position of every - vehicle at every time step. Set to zero if the vehicle is not present - in the network at that time step. - as_array - n_steps x n_veh matrix specifying the speed of every vehicle at every - time step. Set to zero if the vehicle is not present in the network at - that time step. + ndarray + 3d array (n_segments x 2 x 2) containing segments to be plotted. + every inner 2d array is comprised of two 1d arrays representing + [start time, start distance] and [end time, end distance] pairs. + pd.DataFrame + modified trajectory dataframe """ - # import network data from flow params - inflow_edge_len = 100 - premerge = params['net'].additional_params['pre_merge_length'] - postmerge = params['net'].additional_params['post_merge_length'] - - # generate edge starts - edgestarts = { - 'inflow_highway': 0, - 'left': inflow_edge_len + 0.1, - 'center': inflow_edge_len + premerge + 22.6, - 'inflow_merge': inflow_edge_len + premerge + postmerge + 22.6, - 'bottom': 2 * inflow_edge_len + premerge + postmerge + 22.7, - ':left_0': inflow_edge_len, - ':center_0': inflow_edge_len + premerge + 0.1, - ':center_1': inflow_edge_len + premerge + 0.1, - ':bottom_0': 2 * inflow_edge_len + premerge + postmerge + 22.6 - } + segs = data[['time_step', 'distance', 'next_time', 'next_pos']].values.reshape((len(data), 2, 2)) + + return segs, data - # compute the absolute position - for veh_id in data.keys(): - data[veh_id]['abs_pos'] = _get_abs_pos(data[veh_id]['edge'], - data[veh_id]['pos'], edgestarts) - - # prepare the speed and absolute position in a way that is compatible with - # the space-time diagram, and compute the number of vehicles at each step - pos = np.zeros((all_time.shape[0], len(data.keys()))) - speed = np.zeros((all_time.shape[0], len(data.keys()))) - for i, veh_id in enumerate(sorted(data.keys())): - for spd, abs_pos, ti, edge in zip(data[veh_id]['vel'], - data[veh_id]['abs_pos'], - data[veh_id]['time'], - data[veh_id]['edge']): - # avoid vehicles outside the main highway - if edge in ['inflow_merge', 'bottom', ':bottom_0']: - continue - ind = np.where(ti == all_time)[0] - pos[ind, i] = abs_pos - speed[ind, i] = spd - - return pos, speed - - -def _ring_road(data, params, all_time): - r"""Generate position and speed data for the ring road. + +def _ring_road(data): + r"""Generate time and position data for the ring road. Vehicles that reach the top of the plot simply return to the bottom and continue. Parameters ---------- - data : dict of dict - Key = "veh_id": name of the vehicle \n Elements: - - * "time": time step at every sample - * "edge": edge ID at every sample - * "pos": relative position at every sample - * "vel": speed at every sample - params : dict - flow-specific parameters - all_time : array_like - a (n_steps,) vector representing the unique time steps in the - simulation + data : pd.DataFrame + cleaned dataframe of the trajectory data Returns ------- - as_array - n_steps x n_veh matrix specifying the absolute position of every - vehicle at every time step. Set to zero if the vehicle is not present - in the network at that time step. - as_array - n_steps x n_veh matrix specifying the speed of every vehicle at every - time step. Set to zero if the vehicle is not present in the network at - that time step. + ndarray + 3d array (n_segments x 2 x 2) containing segments to be plotted. + every inner 2d array is comprised of two 1d arrays representing + [start time, start distance] and [end time, end distance] pairs. + pd.DataFrame + unmodified trajectory dataframe """ - # import network data from flow params - ring_length = params['net'].additional_params["length"] - junction_length = 0.1 # length of inter-edge junctions - - edgestarts = { - "bottom": 0, - ":right_0": 0.25 * ring_length, - "right": 0.25 * ring_length + junction_length, - ":top_0": 0.5 * ring_length + junction_length, - "top": 0.5 * ring_length + 2 * junction_length, - ":left_0": 0.75 * ring_length + 2 * junction_length, - "left": 0.75 * ring_length + 3 * junction_length, - ":bottom_0": ring_length + 3 * junction_length - } + segs = data[['time_step', 'distance', 'next_time', 'next_pos']].values.reshape((len(data), 2, 2)) + + return segs, data - # compute the absolute position - for veh_id in data.keys(): - data[veh_id]['abs_pos'] = _get_abs_pos(data[veh_id]['edge'], - data[veh_id]['pos'], edgestarts) - # create the output variables - pos = np.zeros((all_time.shape[0], len(data.keys()))) - speed = np.zeros((all_time.shape[0], len(data.keys()))) - for i, veh_id in enumerate(sorted(data.keys())): - for spd, abs_pos, ti in zip(data[veh_id]['vel'], - data[veh_id]['abs_pos'], - data[veh_id]['time']): - ind = np.where(ti == all_time)[0] - pos[ind, i] = abs_pos - speed[ind, i] = spd +def _i210_subnetwork(data): + r"""Generate time and position data for the i210 subnetwork. - return pos, speed + We generate plots for all lanes, so the segments are wrapped in + a dictionary. + Parameters + ---------- + data : pd.DataFrame + cleaned dataframe of the trajectory data + + Returns + ------- + dict < str, np.ndarray > + dictionary of 3d array (n_segments x 2 x 2) containing segments + to be plotted. the dictionary is keyed on lane numbers, with the + values being the 3d array representing the segments. every inner + 2d array is comprised of two 1d arrays representing + [start time, start distance] and [end time, end distance] pairs. + pd.DataFrame + modified trajectory dataframe + """ + # Reset lane numbers that are offset by ramp lanes + offset_edges = set(data[data['lane_id'] == 5]['edge_id'].unique()) + data.loc[data['edge_id'].isin(offset_edges), 'lane_id'] -= 1 -def _figure_eight(data, params, all_time): - r"""Generate position and speed data for the figure eight. + segs = dict() + for lane, df in data.groupby('lane_id'): + segs[lane] = df[['time_step', 'distance', 'next_time', 'next_pos']].values.reshape((len(df), 2, 2)) + + return segs, data + + +def _figure_eight(data): + r"""Generate time and position data for the figure eight. The vehicles traveling towards the intersection from one side will be plotted from the top downward, while the vehicles from the other side will @@ -290,111 +259,207 @@ def _figure_eight(data, params, all_time): Parameters ---------- - data : dict of dict - Key = "veh_id": name of the vehicle \n Elements: - - * "time": time step at every sample - * "edge": edge ID at every sample - * "pos": relative position at every sample - * "vel": speed at every sample - params : dict - flow-specific parameters - all_time : array_like - a (n_steps,) vector representing the unique time steps in the - simulation + data : pd.DataFrame + cleaned dataframe of the trajectory data Returns ------- - as_array - n_steps x n_veh matrix specifying the absolute position of every - vehicle at every time step. Set to zero if the vehicle is not present - in the network at that time step. - as_array - n_steps x n_veh matrix specifying the speed of every vehicle at every - time step. Set to zero if the vehicle is not present in the network at - that time step. + ndarray + 3d array (n_segments x 2 x 2) containing segments to be plotted. + every inner 2d array is comprised of two 1d arrays representing + [start time, start distance] and [end time, end distance] pairs. + pd.DataFrame + unmodified trajectory dataframe """ - # import network data from flow params - net_params = params['net'] - ring_radius = net_params.additional_params['radius_ring'] - ring_edgelen = ring_radius * np.pi / 2. - intersection = 2 * ring_radius - junction = 2.9 + 3.3 * net_params.additional_params['lanes'] - inner = 0.28 - - # generate edge starts - edgestarts = { - 'bottom': inner, - 'top': intersection / 2 + junction + inner, - 'upper_ring': intersection + junction + 2 * inner, - 'right': intersection + 3 * ring_edgelen + junction + 3 * inner, - 'left': 1.5*intersection + 3*ring_edgelen + 2*junction + 3*inner, - 'lower_ring': 2*intersection + 3*ring_edgelen + 2*junction + 4*inner, - ':bottom_0': 0, - ':center_1': intersection / 2 + inner, - ':top_0': intersection + junction + inner, - ':right_0': intersection + 3 * ring_edgelen + junction + 2 * inner, - ':center_0': 1.5*intersection + 3*ring_edgelen + junction + 3*inner, - ':left_0': 2 * intersection + 3*ring_edgelen + 2*junction + 3*inner, - # for aimsun - 'bottom_to_top': intersection / 2 + inner, - 'right_to_left': junction + 3 * inner, - } + segs = data[['time_step', 'distance', 'next_time', 'next_pos']].values.reshape((len(data), 2, 2)) + + return segs, data + - # compute the absolute position - for veh_id in data.keys(): - data[veh_id]['abs_pos'] = _get_abs_pos(data[veh_id]['edge'], - data[veh_id]['pos'], edgestarts) - - # create the output variables - pos = np.zeros((all_time.shape[0], len(data.keys()))) - speed = np.zeros((all_time.shape[0], len(data.keys()))) - for i, veh_id in enumerate(sorted(data.keys())): - for spd, abs_pos, ti in zip(data[veh_id]['vel'], - data[veh_id]['abs_pos'], - data[veh_id]['time']): - ind = np.where(ti == all_time)[0] - pos[ind, i] = abs_pos - speed[ind, i] = spd - - # reorganize data for space-time plot - figure_eight_len = 6*ring_edgelen + 2*intersection + 2*junction + 10*inner - intersection_loc = [edgestarts[':center_1'] + intersection / 2, - edgestarts[':center_0'] + intersection / 2] - pos[pos < intersection_loc[0]] += figure_eight_len - pos[np.logical_and(pos > intersection_loc[0], pos < intersection_loc[1])] \ - += - intersection_loc[1] - pos[pos > intersection_loc[1]] = \ - - pos[pos > intersection_loc[1]] + figure_eight_len + intersection_loc[0] - - return pos, speed - - -def _get_abs_pos(edge, rel_pos, edgestarts): +def _get_abs_pos(df, params): """Compute the absolute positions from edges and relative positions. This is the variable we will ultimately use to plot individual vehicles. Parameters ---------- - edge : list of str - list of edges at every time step - rel_pos : list of float - list of relative positions at every time step - edgestarts : dict - the absolute starting position of every edge + df : pd.DataFrame + dataframe of trajectory data + params : dict + flow-specific parameters Returns ------- - list of float + pd.Series the absolute positive for every sample """ - ret = [] - for edge_i, pos_i in zip(edge, rel_pos): - ret.append(pos_i + edgestarts[edge_i]) + if params['network'] == MergeNetwork: + inflow_edge_len = 100 + premerge = params['net'].additional_params['pre_merge_length'] + postmerge = params['net'].additional_params['post_merge_length'] + + # generate edge starts + edgestarts = { + 'inflow_highway': 0, + 'left': inflow_edge_len + 0.1, + 'center': inflow_edge_len + premerge + 22.6, + 'inflow_merge': inflow_edge_len + premerge + postmerge + 22.6, + 'bottom': 2 * inflow_edge_len + premerge + postmerge + 22.7, + ':left_0': inflow_edge_len, + ':center_0': inflow_edge_len + premerge + 0.1, + ':center_1': inflow_edge_len + premerge + 0.1, + ':bottom_0': 2 * inflow_edge_len + premerge + postmerge + 22.6 + } + elif params['network'] == RingNetwork: + ring_length = params['net'].additional_params["length"] + junction_length = 0.1 # length of inter-edge junctions + + edgestarts = { + "bottom": 0, + ":right_0": 0.25 * ring_length, + "right": 0.25 * ring_length + junction_length, + ":top_0": 0.5 * ring_length + junction_length, + "top": 0.5 * ring_length + 2 * junction_length, + ":left_0": 0.75 * ring_length + 2 * junction_length, + "left": 0.75 * ring_length + 3 * junction_length, + ":bottom_0": ring_length + 3 * junction_length + } + elif params['network'] == FigureEightNetwork: + net_params = params['net'] + ring_radius = net_params.additional_params['radius_ring'] + ring_edgelen = ring_radius * np.pi / 2. + intersection = 2 * ring_radius + junction = 2.9 + 3.3 * net_params.additional_params['lanes'] + inner = 0.28 + + # generate edge starts + edgestarts = { + 'bottom': inner, + 'top': intersection / 2 + junction + inner, + 'upper_ring': intersection + junction + 2 * inner, + 'right': intersection + 3 * ring_edgelen + junction + 3 * inner, + 'left': 1.5 * intersection + 3 * ring_edgelen + 2 * junction + 3 * inner, + 'lower_ring': 2 * intersection + 3 * ring_edgelen + 2 * junction + 4 * inner, + ':bottom_0': 0, + ':center_1': intersection / 2 + inner, + ':top_0': intersection + junction + inner, + ':right_0': intersection + 3 * ring_edgelen + junction + 2 * inner, + ':center_0': 1.5 * intersection + 3 * ring_edgelen + junction + 3 * inner, + ':left_0': 2 * intersection + 3 * ring_edgelen + 2 * junction + 3 * inner, + # for aimsun + 'bottom_to_top': intersection / 2 + inner, + 'right_to_left': junction + 3 * inner, + } + elif params['network'] == HighwayNetwork: + return df['x'] + elif params['network'] == I210SubNetwork: + edgestarts = { + '119257914': -5.0999999999995795, + '119257908#0': 56.49000000018306, + ':300944379_0': 56.18000000000016, + ':300944436_0': 753.4599999999871, + '119257908#1-AddedOnRampEdge': 756.3299999991157, + ':119257908#1-AddedOnRampNode_0': 853.530000000022, + '119257908#1': 856.7699999997207, + ':119257908#1-AddedOffRampNode_0': 1096.4499999999707, + '119257908#1-AddedOffRampEdge': 1099.6899999995558, + ':1686591010_1': 1198.1899999999541, + '119257908#2': 1203.6499999994803, + ':1842086610_1': 1780.2599999999056, + '119257908#3': 1784.7899999996537, + } + else: + edgestarts = defaultdict(float) + + ret = df.apply(lambda x: x['relative_position'] + edgestarts[x['edge_id']], axis=1) + + if params['network'] == FigureEightNetwork: + # reorganize data for space-time plot + figure_eight_len = 6 * ring_edgelen + 2 * intersection + 2 * junction + 10 * inner + intersection_loc = [edgestarts[':center_1'] + intersection / 2, + edgestarts[':center_0'] + intersection / 2] + ret.loc[ret < intersection_loc[0]] += figure_eight_len + ret.loc[(ret > intersection_loc[0]) & (ret < intersection_loc[1])] += -intersection_loc[1] + ret.loc[ret > intersection_loc[1]] = \ + - ret.loc[ret > intersection_loc[1]] + figure_eight_len + intersection_loc[0] return ret +def plot_tsd(ax, df, segs, args, lane=None, ghost_edges=None, ghost_bounds=None): + """Plot the time-space diagram. + + Take the pre-processed segments and other meta-data, then plot all the line segments. + + Parameters + ---------- + ax : matplotlib.axes.Axes + figure axes that will be plotted on + df : pd.DataFrame + data used for axes bounds and speed coloring + segs : list of list of lists + line segments to be plotted, where each segment is a list of two [x,y] pairs + args : dict + parsed arguments + lane : int, optional + lane number to be shown in plot title + ghost_edges : list or set of str + ghost edge names to be greyed out, default None + ghost_bounds : tuple + lower and upper bounds of domain, excluding ghost edges, default None + + Returns + ------- + None + """ + norm = plt.Normalize(args.min_speed, args.max_speed) + + xmin, xmax = df['time_step'].min(), df['time_step'].max() + xbuffer = (xmax - xmin) * 0.025 # 2.5% of range + ymin, ymax = df['distance'].min(), df['distance'].max() + ybuffer = (ymax - ymin) * 0.025 # 2.5% of range + + ax.set_xlim(xmin - xbuffer, xmax + xbuffer) + ax.set_ylim(ymin - ybuffer, ymax + ybuffer) + + lc = LineCollection(segs, cmap=my_cmap, norm=norm) + lc.set_array(df['speed'].values) + lc.set_linewidth(1) + ax.add_collection(lc) + ax.autoscale() + + rects = [] + if ghost_edges: + y_domain_min = df[~df['edge_id'].isin(ghost_edges)]['distance'].min() + y_domain_max = df[~df['edge_id'].isin(ghost_edges)]['distance'].max() + rects.append(Rectangle((xmin, y_domain_min), args.start - xmin, y_domain_max - y_domain_min)) + rects.append(Rectangle((xmin, ymin), xmax - xmin, y_domain_min - ymin)) + rects.append(Rectangle((xmin, y_domain_max), xmax - xmin, ymax - y_domain_max)) + elif ghost_bounds: + rects.append(Rectangle((xmin, ghost_bounds[0]), args.start - xmin, ghost_bounds[1] - ghost_bounds[0])) + rects.append(Rectangle((xmin, ymin), xmax - xmin, ghost_bounds[0] - ymin)) + rects.append(Rectangle((xmin, ghost_bounds[1]), xmax - xmin, ymax - ghost_bounds[1])) + else: + rects.append(Rectangle((xmin, ymin), args.start - xmin, ymax - ymin)) + + if rects: + pc = PatchCollection(rects, facecolor='grey', alpha=0.5, edgecolor=None) + pc.set_zorder(20) + ax.add_collection(pc) + + if lane: + ax.set_title('Time-Space Diagram: Lane {}'.format(lane), fontsize=25) + else: + ax.set_title('Time-Space Diagram', fontsize=25) + ax.set_ylabel('Position (m)', fontsize=20) + ax.set_xlabel('Time (s)', fontsize=20) + plt.xticks(fontsize=18) + plt.yticks(fontsize=18) + + cbar = plt.colorbar(lc, ax=ax, norm=norm) + cbar.set_label('Velocity (m/s)', fontsize=20) + cbar.ax.tick_params(labelsize=18) + + if __name__ == '__main__': # create the parser parser = argparse.ArgumentParser( @@ -404,8 +469,8 @@ def _get_abs_pos(edge, rel_pos, edgestarts): '.json') # required arguments - parser.add_argument('emission_path', type=str, - help='path to the csv file.') + parser.add_argument('trajectory_path', type=str, + help='path to the Flow trajectory csv file.') parser.add_argument('flow_params', type=str, help='path to the flow_params json file.') @@ -416,21 +481,19 @@ def _get_abs_pos(edge, rel_pos, edgestarts): help='rate at which steps are plotted.') parser.add_argument('--max_speed', type=int, default=8, help='The maximum speed in the color range.') + parser.add_argument('--min_speed', type=int, default=0, + help='The minimum speed in the color range.') parser.add_argument('--start', type=float, default=0, help='initial time (in sec) in the plot.') - parser.add_argument('--stop', type=float, default=float('inf'), - help='final time (in sec) in the plot.') args = parser.parse_args() # flow_params is imported as a dictionary - flow_params = get_flow_params(args.flow_params) - - # import data from the emission.csv file - emission_data = import_data_from_emission(args.emission_path) - - # compute the position and speed for all vehicles at all times - pos, speed, time = get_time_space_data(emission_data, flow_params) + if '.json' in args.flow_params: + flow_params = get_flow_params(args.flow_params) + else: + module = __import__("examples.exp_configs.non_rl", fromlist=[args.flow_params]) + flow_params = getattr(module, args.flow_params).flow_params # some plotting parameters cdict = { @@ -440,58 +503,39 @@ def _get_abs_pos(edge, rel_pos, edgestarts): } my_cmap = colors.LinearSegmentedColormap('my_colormap', cdict, 1024) - # perform plotting operation - fig = plt.figure(figsize=(16, 9)) - ax = plt.axes() - norm = plt.Normalize(0, args.max_speed) - cols = [] + # Read trajectory csv into pandas dataframe + traj_df = import_data_from_trajectory(args.trajectory_path, flow_params) - xmin = max(time[0], args.start) - xmax = min(time[-1], args.stop) - xbuffer = (xmax - xmin) * 0.025 # 2.5% of range - ymin, ymax = np.amin(pos), np.amax(pos) - ybuffer = (ymax - ymin) * 0.025 # 2.5% of range - - ax.set_xlim(xmin - xbuffer, xmax + xbuffer) - ax.set_ylim(ymin - ybuffer, ymax + ybuffer) - - for indx_car in range(pos.shape[1]): - unique_car_pos = pos[:, indx_car] - - # discontinuity from wraparound - disc = np.where(np.abs(np.diff(unique_car_pos)) >= 10)[0] + 1 - unique_car_time = np.insert(time, disc, np.nan) - unique_car_pos = np.insert(unique_car_pos, disc, np.nan) - unique_car_speed = np.insert(speed[:, indx_car], disc, np.nan) + # Convert df data into segments for plotting + segs, traj_df = get_time_space_data(traj_df, flow_params) - points = np.array( - [unique_car_time, unique_car_pos]).T.reshape(-1, 1, 2) - segments = np.concatenate([points[:-1], points[1:]], axis=1) - lc = LineCollection(segments, cmap=my_cmap, norm=norm) + if flow_params['network'] == I210SubNetwork: + nlanes = traj_df['lane_id'].nunique() + fig = plt.figure(figsize=(16, 9*nlanes)) - # Set the values used for color mapping - lc.set_array(unique_car_speed) - lc.set_linewidth(1.75) - cols.append(lc) + for lane, df in traj_df.groupby('lane_id'): + ax = plt.subplot(nlanes, 1, lane+1) - plt.title(args.title, fontsize=25) - plt.ylabel('Position (m)', fontsize=20) - plt.xlabel('Time (s)', fontsize=20) + plot_tsd(ax, df, segs[lane], args, int(lane+1), ghost_edges={'ghost0', '119257908#3'}) + plt.tight_layout() + else: + # perform plotting operation + fig = plt.figure(figsize=(16, 9)) + ax = plt.axes() - for col in cols: - line = ax.add_collection(col) - cbar = plt.colorbar(line, ax=ax) - cbar.set_label('Velocity (m/s)', fontsize=20) - cbar.ax.tick_params(labelsize=18) - - plt.xticks(fontsize=18) - plt.yticks(fontsize=18) + if flow_params['network'] == HighwayNetwork: + plot_tsd(ax, traj_df, segs, args, ghost_bounds=(500, 2300)) + else: + plot_tsd(ax, traj_df, segs, args) ########################################################################### # Note: For MergeNetwork only # if flow_params['network'] == 'MergeNetwork': # - plt.plot(time, [0] * pos.shape[0], linewidth=3, color="white") # - plt.plot(time, [-0.1] * pos.shape[0], linewidth=3, color="white") # + plt.plot([df['time_step'].min(), df['time_step'].max()], + [0, 0], linewidth=3, color="white") # + plt.plot([df['time_step'].min(), df['time_step'].max()], + [-0.1, -0.1], linewidth=3, color="white") # ########################################################################### - plt.show() + outfile = args.trajectory_path.replace('csv', 'png') + plt.savefig(outfile) diff --git a/flow/visualize/visualizer_rllib.py b/flow/visualize/visualizer_rllib.py index f9a94d8b2..c1dd83193 100644 --- a/flow/visualize/visualizer_rllib.py +++ b/flow/visualize/visualizer_rllib.py @@ -13,7 +13,6 @@ """ import argparse -from datetime import datetime import gym import numpy as np import os @@ -75,6 +74,10 @@ def visualizer_rllib(args): sim_params = flow_params['sim'] setattr(sim_params, 'num_clients', 1) + # for hacks for old pkl files TODO: remove eventually + if not hasattr(sim_params, 'use_ballistic'): + sim_params.use_ballistic = False + # Determine agent and checkpoint config_run = config['env_config']['run'] if 'run' in config['env_config'] \ else None @@ -110,15 +113,13 @@ def visualizer_rllib(args): sim_params.render = 'drgb' sim_params.pxpm = 4 elif args.render_mode == 'sumo_gui': - sim_params.render = True - print('NOTE: With render mode {}, an extra instance of the SUMO GUI ' - 'will display before the GUI for visualizing the result. Click ' - 'the green Play arrow to continue.'.format(args.render_mode)) + sim_params.render = False # will be set to True below elif args.render_mode == 'no_render': sim_params.render = False if args.save_render: - sim_params.render = 'drgb' - sim_params.pxpm = 4 + if args.render_mode != 'sumo_gui': + sim_params.render = 'drgb' + sim_params.pxpm = 4 sim_params.save_render = True # Create and register a gym+rllib env @@ -159,10 +160,13 @@ def visualizer_rllib(args): else: env = gym.make(env_name) + if args.render_mode == 'sumo_gui': + env.sim_params.render = True # set to True after initializing agent and env + if multiagent: rets = {} # map the agent id to its policy - policy_map_fn = config['multiagent']['policy_mapping_fn'].func + policy_map_fn = config['multiagent']['policy_mapping_fn'] for key in config['multiagent']['policies'].keys(): rets[key] = [] else: @@ -173,7 +177,7 @@ def visualizer_rllib(args): if multiagent: state_init = {} # map the agent id to its policy - policy_map_fn = config['multiagent']['policy_mapping_fn'].func + policy_map_fn = config['multiagent']['policy_mapping_fn'] size = config['model']['lstm_cell_size'] for key in config['multiagent']['policies'].keys(): state_init[key] = [np.zeros(size, np.float32), @@ -186,8 +190,9 @@ def visualizer_rllib(args): else: use_lstm = False - env.restart_simulation( - sim_params=sim_params, render=sim_params.render) + # if restart_instance, don't restart here because env.reset will restart later + if not sim_params.restart_instance: + env.restart_simulation(sim_params=sim_params, render=sim_params.render) # Simulate and collect metrics final_outflows = [] @@ -203,7 +208,12 @@ def visualizer_rllib(args): ret = 0 for _ in range(env_params.horizon): vehicles = env.unwrapped.k.vehicle - vel.append(np.mean(vehicles.get_speed(vehicles.get_ids()))) + speeds = vehicles.get_speed(vehicles.get_ids()) + + # only include non-empty speeds + if speeds: + vel.append(np.mean(speeds)) + if multiagent: action = {} for agent_id in state.keys(): @@ -306,26 +316,13 @@ def visualizer_rllib(args): # convert the emission file into a csv file emission_to_csv(emission_path) + # print the location of the emission csv file + emission_path_csv = emission_path[:-4] + ".csv" + print("\nGenerated emission file at " + emission_path_csv) + # delete the .xml version of the emission file os.remove(emission_path) - # if we wanted to save the render, here we create the movie - if args.save_render: - dirs = os.listdir(os.path.expanduser('~')+'/flow_rendering') - # Ignore hidden files - dirs = [d for d in dirs if d[0] != '.'] - dirs.sort(key=lambda date: datetime.strptime(date, "%Y-%m-%d-%H%M%S")) - recent_dir = dirs[-1] - # create the movie - movie_dir = os.path.expanduser('~') + '/flow_rendering/' + recent_dir - save_dir = os.path.expanduser('~') + '/flow_movies' - if not os.path.exists(save_dir): - os.mkdir(save_dir) - os_cmd = "cd " + movie_dir + " && ffmpeg -i frame_%06d.png" - os_cmd += " -pix_fmt yuv420p " + dirs[-1] + ".mp4" - os_cmd += "&& cp " + dirs[-1] + ".mp4 " + save_dir + "/" - os.system(os_cmd) - def create_parser(): """Create the parser to capture CLI arguments.""" diff --git a/requirements.txt b/requirements.txt index 483392a76..a4f6f83f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ gym==0.14.0 -numpy==1.16.0 +numpy==1.18.4 scipy==1.1.0 -lxml==4.2.4 +lxml==4.4.1 pyprind==2.11.2 nose2==0.8.0 six==1.11.0 @@ -9,18 +9,24 @@ path.py joblib==0.10.3 python-dateutil==2.7.3 cached_property -cloudpickle==0.5.3 pyglet==1.3.2 -matplotlib==3.0.0 +matplotlib==3.1.0 imutils==0.5.1 numpydoc -ray==0.7.3 +ray==0.8.0 opencv-python dill lz4 setproctitle psutil opencv-python -boto3==1.4.8 +boto3==1.10.45 redis~=2.10.6 pandas==0.24.2 +plotly==2.4.0 +tabulate +tensorflow==1.15.2 +awscli==1.16.309 +torch==1.4.0 +pytz +tensorboardX diff --git a/scripts/ray_autoscale.yaml b/scripts/ray_autoscale.yaml index e6dab39bd..27ac0898e 100644 --- a/scripts/ray_autoscale.yaml +++ b/scripts/ray_autoscale.yaml @@ -68,9 +68,13 @@ worker_nodes: setup_commands: - cd flow && git fetch && git checkout origin/master + - pip install boto3==1.10.45 # 1.4.8 adds InstanceMarketOptions + - pip install awscli==1.16.309 + - pip install pytz + - pip install numpy==1.18.4 + - ./flow/scripts/setup_sumo_ubuntu1604.sh -head_setup_commands: - - pip install boto3==1.4.8 # 1.4.8 adds InstanceMarketOptions +head_setup_commands: [] # Custom commands that will be run on worker nodes after common setup. worker_setup_commands: [] diff --git a/setup.py b/setup.py index f9508507e..70b894da2 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ class build_ext(_build_ext.build_ext): def run(self): """Install traci wheels.""" subprocess.check_call( - ['pip', 'install', + ['python3','-m','pip', 'install', 'https://akreidieh.s3.amazonaws.com/sumo/flow-0.4.0/' 'sumotools-0.4.0-py3-none-any.whl']) diff --git a/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1 b/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1 index 0693ed4b6..d346e9dc5 100644 Binary files a/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1 and b/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1 differ diff --git a/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1.tune_metadata b/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1.tune_metadata index 7eef2ef15..febe7b205 100644 Binary files a/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1.tune_metadata and b/tests/data/rllib_data/multi_agent/checkpoint_1/checkpoint-1.tune_metadata differ diff --git a/tests/data/rllib_data/multi_agent/params.json b/tests/data/rllib_data/multi_agent/params.json index 01089f730..1aa9a114c 100644 --- a/tests/data/rllib_data/multi_agent/params.json +++ b/tests/data/rllib_data/multi_agent/params.json @@ -8,17 +8,19 @@ "on_sample_end": null, "on_train_result": null }, - "clip_actions": false, + "clip_actions": true, "clip_param": 0.3, "clip_rewards": null, "collect_metrics_timeout": 180, "compress_observations": false, "custom_resources_per_worker": {}, + "eager": false, + "eager_tracing": false, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, - "env": "MultiWaveAttenuationPOEnv-v0", + "env": "MultiAgentAccelPOEnv-v1", "env_config": { - "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 230,\n 230\n ],\n \"target_velocity\": 4\n },\n \"clip_actions\": true,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"MultiWaveAttenuationPOEnv\",\n \"exp_tag\": \"lord_of_numrings1\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 20.0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"custom\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 230,\n \"num_rings\": 1,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"MultiRingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human_0\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl_0\"\n }\n ]\n}", + "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 3,\n \"max_decel\": 3,\n \"sort_vehicles\": false,\n \"target_velocity\": 20\n },\n \"clip_actions\": true,\n \"evaluate\": false,\n \"horizon\": 1500,\n \"sims_per_step\": 1,\n \"warmup_steps\": 0\n },\n \"env_name\": \"flow.envs.multiagent.ring.accel.MultiAgentAccelPOEnv\",\n \"exp_tag\": \"multiagent_figure_eight\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"radius_ring\": 30,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.figure_eight.FigureEightNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1,\n \"use_ballistic\": false\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 1.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 6,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human_0\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 3,\n \"carFollowModel\": \"IDM\",\n \"decel\": 3,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl_0\"\n },\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 1.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 6,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human_1\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 3,\n \"carFollowModel\": \"IDM\",\n \"decel\": 3,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl_1\"\n }\n ]\n}", "run": "PPO" }, "evaluation_config": {}, @@ -26,7 +28,7 @@ "evaluation_num_episodes": 10, "gamma": 0.999, "grad_clip": null, - "horizon": 3000, + "horizon": 1500, "ignore_worker_failures": false, "input": "sampler", "input_evaluation": [ @@ -34,27 +36,31 @@ "wis" ], "kl_coeff": 0.2, - "kl_target": 0.01, - "lambda": 1.0, + "kl_target": 0.02, + "lambda": 0.97, "local_tf_session_args": { "inter_op_parallelism_threads": 8, "intra_op_parallelism_threads": 8 }, - "log_level": "INFO", + "log_level": "WARN", "log_sys_usage": true, - "lr": 1e-05, + "lr": 5e-05, "lr_schedule": null, + "memory": 0, + "memory_per_worker": 0, "metrics_smoothing_episodes": 100, "min_iter_time_s": 0, "model": { "conv_activation": "relu", "conv_filters": null, + "custom_action_dist": null, "custom_model": null, "custom_options": {}, "custom_preprocessor": null, "dim": 84, "fcnet_activation": "tanh", "fcnet_hiddens": [ + 32, 32, 32 ], @@ -75,23 +81,25 @@ "policies": { "av": [ "", - "Box(3,)", + "Box(6,)", "Box(1,)", {} ] }, - "policies_to_train": [ - "av" - ], - "policy_mapping_fn": "tune.function(.policy_mapping_fn at 0x7fda132e6c80>)" + "policies_to_train": null, + "policy_mapping_fn": "" }, + "no_done_at_end": false, + "no_eager_on_workers": false, "num_cpus_for_driver": 1, "num_cpus_per_worker": 1, "num_envs_per_worker": 1, "num_gpus": 0, "num_gpus_per_worker": 0, - "num_sgd_iter": 30, + "num_sgd_iter": 10, "num_workers": 2, + "object_store_memory": 0, + "object_store_memory_per_worker": 0, "observation_filter": "NoFilter", "optimizer": {}, "output": null, @@ -110,7 +118,7 @@ "sgd_minibatch_size": 128, "shuffle_buffer_size": 0, "shuffle_sequences": true, - "simple_optimizer": true, + "simple_optimizer": false, "soft_horizon": false, "synchronize_filters": true, "tf_session_args": { @@ -126,7 +134,7 @@ "log_device_placement": false }, "timesteps_per_iteration": 0, - "train_batch_size": 60000, + "train_batch_size": 30000, "use_gae": true, "vf_clip_param": 10.0, "vf_loss_coeff": 1.0, diff --git a/tests/data/rllib_data/multi_agent/params.pkl b/tests/data/rllib_data/multi_agent/params.pkl index cd832aa1c..192cf7558 100644 Binary files a/tests/data/rllib_data/multi_agent/params.pkl and b/tests/data/rllib_data/multi_agent/params.pkl differ diff --git a/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1 b/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1 index f8a7e8976..b7ae94640 100644 Binary files a/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1 and b/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1 differ diff --git a/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1.tune_metadata b/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1.tune_metadata index e83b72aea..55b72be28 100644 Binary files a/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1.tune_metadata and b/tests/data/rllib_data/single_agent/checkpoint_1/checkpoint-1.tune_metadata differ diff --git a/tests/data/rllib_data/single_agent/params.json b/tests/data/rllib_data/single_agent/params.json index c5e605ef4..c93580225 100644 --- a/tests/data/rllib_data/single_agent/params.json +++ b/tests/data/rllib_data/single_agent/params.json @@ -8,17 +8,19 @@ "on_sample_end": null, "on_train_result": null }, - "clip_actions": false, + "clip_actions": true, "clip_param": 0.3, "clip_rewards": null, "collect_metrics_timeout": 180, "compress_observations": false, "custom_resources_per_worker": {}, + "eager": false, + "eager_tracing": false, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, - "env": "WaveAttenuationPOEnv-v0", + "env": "AccelEnv-v0", "env_config": { - "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", + "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 3,\n \"max_decel\": 3,\n \"sort_vehicles\": false,\n \"target_velocity\": 20\n },\n \"clip_actions\": true,\n \"evaluate\": false,\n \"horizon\": 1500,\n \"sims_per_step\": 1,\n \"warmup_steps\": 0\n },\n \"env_name\": \"flow.envs.ring.accel.AccelEnv\",\n \"exp_tag\": \"singleagent_figure_eight\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"radius_ring\": 30,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.figure_eight.FigureEightNetwork\",\n \"sim\": {\n \"color_by_speed\": false,\n \"emission_path\": null,\n \"force_color_update\": false,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1,\n \"use_ballistic\": false\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 1.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 13,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 1.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 1\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO" }, "evaluation_config": {}, @@ -26,7 +28,7 @@ "evaluation_num_episodes": 10, "gamma": 0.999, "grad_clip": null, - "horizon": 3000, + "horizon": 1500, "ignore_worker_failures": false, "input": "sampler", "input_evaluation": [ @@ -40,23 +42,27 @@ "inter_op_parallelism_threads": 8, "intra_op_parallelism_threads": 8 }, - "log_level": "INFO", + "log_level": "WARN", "log_sys_usage": true, "lr": 5e-05, "lr_schedule": null, + "memory": 0, + "memory_per_worker": 0, "metrics_smoothing_episodes": 100, "min_iter_time_s": 0, "model": { "conv_activation": "relu", "conv_filters": null, + "custom_action_dist": null, "custom_model": null, "custom_options": {}, "custom_preprocessor": null, "dim": 84, "fcnet_activation": "tanh", "fcnet_hiddens": [ - 3, - 3 + 32, + 32, + 32 ], "framestack": true, "free_log_std": false, @@ -76,6 +82,8 @@ "policies_to_train": null, "policy_mapping_fn": null }, + "no_done_at_end": false, + "no_eager_on_workers": false, "num_cpus_for_driver": 1, "num_cpus_per_worker": 1, "num_envs_per_worker": 1, @@ -83,6 +91,8 @@ "num_gpus_per_worker": 0, "num_sgd_iter": 10, "num_workers": 2, + "object_store_memory": 0, + "object_store_memory_per_worker": 0, "observation_filter": "NoFilter", "optimizer": {}, "output": null, @@ -117,7 +127,7 @@ "log_device_placement": false }, "timesteps_per_iteration": 0, - "train_batch_size": 60000, + "train_batch_size": 30000, "use_gae": true, "vf_clip_param": 10.0, "vf_loss_coeff": 1.0, diff --git a/tests/data/rllib_data/single_agent/params.pkl b/tests/data/rllib_data/single_agent/params.pkl index 511d34343..e69753b7f 100644 Binary files a/tests/data/rllib_data/single_agent/params.pkl and b/tests/data/rllib_data/single_agent/params.pkl differ diff --git a/tests/fast_tests/test_collisions.py b/tests/fast_tests/test_collisions.py index dbb9fdcca..ec0e41f9a 100644 --- a/tests/fast_tests/test_collisions.py +++ b/tests/fast_tests/test_collisions.py @@ -50,27 +50,27 @@ def test_collide(self): net_params = NetParams(additional_params=additional_net_params) - env, _ = traffic_light_grid_mxn_exp_setup( + env, _, flow_params = traffic_light_grid_mxn_exp_setup( row_num=1, col_num=1, sim_params=sim_params, vehicles=vehicles, net_params=net_params) + # create an experiment object + exp = Experiment(flow_params) + env = exp.env # go through the env and set all the lights to green for i in range(env.rows * env.cols): env.k.traffic_light.set_state( node_id='center' + str(i), state="gggggggggggg") - # instantiate an experiment class - exp = Experiment(env) - - exp.run(50, 50) + exp.run(50) def test_collide_inflows(self): """Tests collisions in the presence of inflows.""" # create the environment and network classes for a ring road - sim_params = SumoParams(sim_step=1, render=False) + sim_params = SumoParams(sim_step=1, render=False, seed=40) total_vehicles = 0 vehicles = VehicleParams() vehicles.add( @@ -111,22 +111,23 @@ def test_collide_inflows(self): inflows=inflows, additional_params=additional_net_params) - env, _ = traffic_light_grid_mxn_exp_setup( + _, _, flow_params = traffic_light_grid_mxn_exp_setup( row_num=1, col_num=1, sim_params=sim_params, vehicles=vehicles, net_params=net_params) + # instantiate an experiment class + exp = Experiment(flow_params) + env = exp.env + # go through the env and set all the lights to green for i in range(env.rows * env.cols): env.k.traffic_light.set_state( node_id='center' + str(i), state="gggggggggggg") - # instantiate an experiment class - exp = Experiment(env) - - exp.run(50, 50) + exp.run(50) if __name__ == '__main__': diff --git a/tests/fast_tests/test_controllers.py b/tests/fast_tests/test_controllers.py index a521a095e..bef765396 100644 --- a/tests/fast_tests/test_controllers.py +++ b/tests/fast_tests/test_controllers.py @@ -7,8 +7,9 @@ from flow.controllers.routing_controllers import ContinuousRouter from flow.controllers.car_following_models import IDMController, \ - OVMController, BCMController, LinearOVM, CFMController, LACController -from flow.controllers import FollowerStopper, PISaturation + OVMController, BCMController, LinearOVM, CFMController, LACController, \ + GippsController, BandoFTLController +from flow.controllers import FollowerStopper, PISaturation, NonLocalFollowerStopper from tests.setup_scripts import ring_road_exp_setup import os import numpy as np @@ -44,7 +45,7 @@ def setUp(self): num_vehicles=5) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup(vehicles=vehicles) + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) def tearDown(self): # terminate the traci instance @@ -93,7 +94,7 @@ def setUp(self): num_vehicles=5) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup(vehicles=vehicles) + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) def tearDown(self): # terminate the traci instance @@ -148,7 +149,7 @@ def setUp(self): num_vehicles=5) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup(vehicles=vehicles) + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) def tearDown(self): # terminate the traci instance @@ -198,7 +199,7 @@ def setUp(self): num_vehicles=5) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup(vehicles=vehicles) + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) def tearDown(self): # terminate the traci instance @@ -245,7 +246,7 @@ def setUp(self): num_vehicles=5) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup(vehicles=vehicles) + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) def tearDown(self): # terminate the traci instance @@ -311,14 +312,15 @@ def setUp_failsafe(self, vehicles): initial_config = InitialConfig(bunching=10) # create the environment and network classes for a ring road - env, _ = ring_road_exp_setup( + _, _, flow_params = ring_road_exp_setup( vehicles=vehicles, env_params=env_params, net_params=net_params, initial_config=initial_config) + flow_params['env'].horizon = 200 # instantiate an experiment class - self.exp = Experiment(env) + self.exp = Experiment(flow_params) def tearDown_failsafe(self): # free data used by the class @@ -338,7 +340,7 @@ def test_no_crash_OVM(self): self.setUp_failsafe(vehicles=vehicles) # run the experiment, see if it fails - self.exp.run(1, 200) + self.exp.run(1) self.tearDown_failsafe() @@ -355,7 +357,7 @@ def test_no_crash_LinearOVM(self): self.setUp_failsafe(vehicles=vehicles) # run the experiment, see if it fails - self.exp.run(1, 200) + self.exp.run(1) self.tearDown_failsafe() @@ -380,7 +382,7 @@ def test_no_crash_OVM(self): self.setUp_failsafe(vehicles=vehicles) # run the experiment, see if it fails - self.exp.run(1, 200) + self.exp.run(1) self.tearDown_failsafe() @@ -398,7 +400,176 @@ def test_no_crash_LinearOVM(self): self.setUp_failsafe(vehicles=vehicles) # run the experiment, see if it fails - self.exp.run(1, 200) + self.exp.run(1) + + self.tearDown_failsafe() + + +class TestFeasibleAccelFailsafe(TestInstantaneousFailsafe): + """ + Tests that the feasible accel failsafe of the base acceleration controller + does not fail under extreme conditions. + """ + + def test_no_crash_OVM(self): + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(OVMController, { + "fail_safe": "feasible_accel" + }), + routing_controller=(ContinuousRouter, {}), + num_vehicles=10, + ) + + self.setUp_failsafe(vehicles=vehicles) + + # run the experiment, see if it fails + self.exp.run(1) + + self.tearDown_failsafe() + + def test_no_crash_LinearOVM(self): + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(LinearOVM, { + "fail_safe": "feasible_accel" + }), + routing_controller=(ContinuousRouter, {}), + num_vehicles=10, + ) + + self.setUp_failsafe(vehicles=vehicles) + + # run the experiment, see if it fails + self.exp.run(1) + + self.tearDown_failsafe() + + +class TestObeySpeedLimitFailsafe(TestInstantaneousFailsafe): + """ + Tests that the obey speed limit failsafe of the base acceleration controller + does not fail under extreme conditions. + """ + + def test_no_crash_OVM(self): + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(OVMController, { + "fail_safe": "obey_speed_limit" + }), + routing_controller=(ContinuousRouter, {}), + num_vehicles=10, + ) + + self.setUp_failsafe(vehicles=vehicles) + + # run the experiment, see if it fails + self.exp.run(1) + + self.tearDown_failsafe() + + def test_no_crash_LinearOVM(self): + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(LinearOVM, { + "fail_safe": "obey_speed_limit" + }), + routing_controller=(ContinuousRouter, {}), + num_vehicles=10, + ) + + self.setUp_failsafe(vehicles=vehicles) + + # run the experiment, see if it fails + self.exp.run(1) + + self.tearDown_failsafe() + + +class TestBrokenFailsafe(TestInstantaneousFailsafe): + """ + Tests that the failsafe logic triggers exceptions when instantiated + incorrectly. + """ + + def test_invalid_failsafe_string(self): + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(OVMController, { + "fail_safe": "default" + }), + routing_controller=(ContinuousRouter, {}), + num_vehicles=10, + ) + + additional_env_params = { + "target_velocity": 8, + "max_accel": 3, + "max_decel": 3, + "sort_vehicles": False + } + env_params = EnvParams(additional_params=additional_env_params) + + additional_net_params = { + "length": 100, + "lanes": 1, + "speed_limit": 30, + "resolution": 40 + } + net_params = NetParams(additional_params=additional_net_params) + + initial_config = InitialConfig(bunching=10) + + # create the environment and network classes, see that it raises ValueError + with self.assertRaises(ValueError): + ring_road_exp_setup(vehicles=vehicles, + env_params=env_params, + net_params=net_params, + initial_config=initial_config) + + self.tearDown_failsafe() + + def test_invalid_failsafe_type(self): + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(LinearOVM, { + "fail_safe": True + }), + routing_controller=(ContinuousRouter, {}), + num_vehicles=10, + ) + + additional_env_params = { + "target_velocity": 8, + "max_accel": 3, + "max_decel": 3, + "sort_vehicles": False + } + env_params = EnvParams(additional_params=additional_env_params) + + additional_net_params = { + "length": 100, + "lanes": 1, + "speed_limit": 30, + "resolution": 40 + } + net_params = NetParams(additional_params=additional_net_params) + + initial_config = InitialConfig(bunching=10) + + # create the environment and network classes, see that it raises ValueError + with self.assertRaises(ValueError): + ring_road_exp_setup(vehicles=vehicles, + env_params=env_params, + net_params=net_params, + initial_config=initial_config) self.tearDown_failsafe() @@ -420,7 +591,7 @@ def setUp(self): net_params = NetParams(additional_params=additional_net_params) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup(net_params=net_params) + self.env, _, _ = ring_road_exp_setup(net_params=net_params) def tearDown(self): # terminate the traci instance @@ -478,7 +649,7 @@ def setUp(self): num_vehicles=5) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup(vehicles=vehicles) + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) def tearDown(self): # terminate the traci instance @@ -531,6 +702,78 @@ def test_find_intersection_dist(self): ids[0]).get_action(self.env)) +class TestNonLocalFollowerStopper(unittest.TestCase): + + """Makes sure that the nonlocal follower stopper runs.""" + + def setUp(self): + # add a few vehicles to the network using the requested model + # also make sure that the input params are what is expected + contr_params = {"v_des": 7.5} + + vehicles = VehicleParams() + vehicles.add( + veh_id="test_0", + acceleration_controller=(NonLocalFollowerStopper, contr_params), + routing_controller=(ContinuousRouter, {}), + car_following_params=SumoCarFollowingParams( + accel=20, decel=5), + num_vehicles=5) + + # create the environment and network classes for a ring road + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) + + def tearDown(self): + # terminate the traci instance + self.env.terminate() + + # free data used by the class + self.env = None + + def test_get_action(self): + self.env.reset() + ids = self.env.k.vehicle.get_ids() + + test_headways = [5, 10, 15, 20, 25] + test_speeds = [5, 7.5, 7.5, 8, 7] + for i, veh_id in enumerate(ids): + self.env.k.vehicle.set_headway(veh_id, test_headways[i]) + self.env.k.vehicle.test_set_speed(veh_id, test_speeds[i]) + + requested_accel = [ + self.env.k.vehicle.get_acc_controller(veh_id).get_action(self.env) + for veh_id in ids + ] + + expected_accel = [-3.33333333333333, -5.0, -5.0, -10.0, 0.0] + + np.testing.assert_array_almost_equal(requested_accel, expected_accel) + + def test_find_intersection_dist(self): + self.env.reset() + ids = self.env.k.vehicle.get_ids() + + test_edges = ["", "center"] + for i, veh_id in enumerate(ids): + if i < 2: + self.env.k.vehicle.test_set_edge(veh_id, test_edges[i]) + + requested = [ + self.env.k.vehicle.get_acc_controller( + veh_id).find_intersection_dist(self.env) + for veh_id in ids + ] + + expected = [-10, 0, 23.1, 34.7, 46.3] + + np.testing.assert_array_almost_equal(requested, expected) + + # we also check that the accel value is None when this value is + # negative + self.assertIsNone(self.env.k.vehicle.get_acc_controller( + ids[0]).get_action(self.env)) + + class TestPISaturation(unittest.TestCase): """ @@ -553,7 +796,7 @@ def setUp(self): num_vehicles=5) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup(vehicles=vehicles) + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) def tearDown(self): # terminate the traci instance @@ -608,7 +851,7 @@ def setUp(self): num_vehicles=5) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup(vehicles=vehicles) + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) def tearDown(self): # terminate the traci instance @@ -635,5 +878,115 @@ def test_get_action(self): np.testing.assert_array_almost_equal(requested_accel, expected_accel) +class TestGippsController(unittest.TestCase): + """ + Tests that the Gipps Controller returning mathematically accurate values. + """ + + def setUp(self): + # add a few vehicles to the network using the requested model + # also make sure that the input params are what is expected + contr_params = { + "v0": 30, + "acc": 1.5, + "b": -1, + "b_l": -1, + "s0": 2, + "tau": 1, + "delay": 0, + "noise": 0, + } + + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(GippsController, contr_params), + routing_controller=(ContinuousRouter, {}), + car_following_params=SumoCarFollowingParams( + accel=15, decel=5), + num_vehicles=5) + + # create the environment and network classes for a ring road + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) + + def tearDown(self): + # terminate the traci instance + self.env.terminate() + + # free data used by the class + self.env = None + + def test_get_action(self): + self.env.reset() + ids = self.env.k.vehicle.get_ids() + + test_headways = [2, 4, 6, 8, 10] + for i, veh_id in enumerate(ids): + self.env.k.vehicle.set_headway(veh_id, test_headways[i]) + + requested_accel = [ + self.env.k.vehicle.get_acc_controller(veh_id).get_action(self.env) + for veh_id in ids + ] + + expected_accel = [0., 5.929271, 5.929271, 5.929271, 5.929271] + + np.testing.assert_array_almost_equal(requested_accel, expected_accel) + + +class TestBandoFTLController(unittest.TestCase): + """ + Tests that the Bando Controller returning mathematically accurate values. + """ + + def setUp(self): + # add a few vehicles to the network using the requested model + # also make sure that the input params are what is expected + contr_params = { + "alpha": .5, + "beta": 20, + "h_st": 2, + "h_go": 10, + "v_max": 32, + "want_max_accel": False, + } + + vehicles = VehicleParams() + vehicles.add( + veh_id="test", + acceleration_controller=(BandoFTLController, contr_params), + routing_controller=(ContinuousRouter, {}), + car_following_params=SumoCarFollowingParams( + accel=15, decel=5), + num_vehicles=5) + + # create the environment and network classes for a ring road + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) + + def tearDown(self): + # terminate the traci instance + self.env.terminate() + + # free data used by the class + self.env = None + + def test_get_action(self): + self.env.reset() + ids = self.env.k.vehicle.get_ids() + + test_headways = [2, 4, 6, 8, 10] + for i, veh_id in enumerate(ids): + self.env.k.vehicle.set_headway(veh_id, test_headways[i]) + + requested_accel = [ + self.env.k.vehicle.get_acc_controller(veh_id).get_action(self.env) + for veh_id in ids + ] + + expected_accel = [1.649129, 7.853475, 14.057821, 15.70695, 15.959713] + + np.testing.assert_array_almost_equal(requested_accel, expected_accel) + + if __name__ == '__main__': unittest.main() diff --git a/tests/fast_tests/test_environment_base_class.py b/tests/fast_tests/test_environment_base_class.py index b2e529b3a..ee815393c 100644 --- a/tests/fast_tests/test_environment_base_class.py +++ b/tests/fast_tests/test_environment_base_class.py @@ -13,8 +13,9 @@ from tests.setup_scripts import ring_road_exp_setup, highway_exp_setup import os -import numpy as np import gym.spaces as spaces +from gym.spaces.box import Box +import numpy as np os.environ["TEST_FLAG"] = "True" @@ -25,6 +26,41 @@ YELLOW = (255, 255, 0) +class TestFailRLActionsEnv(Env): + """Test environment designed to fail _apply_rl_actions not-implemented test.""" + + @property + def action_space(self): + """See parent class.""" + return Box(low=0, high=0, shape=(0,), dtype=np.float32) # pragma: no cover + + @property + def observation_space(self): + """See parent class.""" + return Box(low=0, high=0, shape=(0,), dtype=np.float32) # pragma: no cover + + def get_state(self, **kwargs): + """See class definition.""" + return np.array([]) # pragma: no cover + + +class TestFailGetStateEnv(Env): + """Test environment designed to fail get_state not-implemented test.""" + + @property + def action_space(self): + """See parent class.""" + return Box(low=0, high=0, shape=(0,), dtype=np.float32) # pragma: no cover + + @property + def observation_space(self): + """See parent class.""" + return Box(low=0, high=0, shape=(0,), dtype=np.float32) # pragma: no cover + + def _apply_rl_actions(self, rl_actions): + return # pragma: no cover + + class TestShuffle(unittest.TestCase): """ Tests that, at resets, the ordering of vehicles changes while the starting @@ -47,7 +83,7 @@ def setUp(self): initial_config = InitialConfig(x0=5, shuffle=True) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup( + self.env, _, _ = ring_road_exp_setup( env_params=env_params, initial_config=initial_config, vehicles=vehicles) @@ -87,7 +123,7 @@ def setUp(self): sim_params = SumoParams() # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup(sim_params=sim_params) + self.env, _, _ = ring_road_exp_setup(sim_params=sim_params) def tearDown(self): # terminate the traci instance @@ -133,7 +169,7 @@ def setUp(self): num_vehicles=5) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup( + self.env, _, _ = ring_road_exp_setup( net_params=net_params, env_params=env_params, vehicles=vehicles) def tearDown(self): @@ -264,7 +300,7 @@ def test_it_works(self): # than one env_params = EnvParams( warmup_steps=warmup_step, additional_params=ADDITIONAL_ENV_PARAMS) - env, _ = ring_road_exp_setup(env_params=env_params) + env, _, _ = ring_road_exp_setup(env_params=env_params) # time before running a reset t1 = 0 @@ -289,7 +325,7 @@ def test_it_works(self): env_params = EnvParams( sims_per_step=sims_per_step, additional_params=ADDITIONAL_ENV_PARAMS) - env, _ = ring_road_exp_setup(env_params=env_params) + env, _, _ = ring_road_exp_setup(env_params=env_params) env.reset() # time before running a step @@ -311,28 +347,34 @@ class TestAbstractMethods(unittest.TestCase): """ def setUp(self): - env, network = ring_road_exp_setup() - sim_params = SumoParams() # FIXME: make ambiguous - env_params = EnvParams() - self.env = Env(sim_params=sim_params, - env_params=env_params, - network=network) + self.env, self.network, _ = ring_road_exp_setup() + self.sim_params = SumoParams() # FIXME: make ambiguous + self.env_params = EnvParams() - def tearDown(self): - self.env.terminate() - self.env = None + def test_abstract_base_class(self): + """Checks that instantiating abstract base class raises an error.""" + with self.assertRaises(TypeError): + Env(sim_params=self.sim_params, + env_params=self.env_params, + network=self.network) def test_get_state(self): - """Checks that get_state raises an error.""" - self.assertRaises(NotImplementedError, self.env.get_state) - - def test_compute_reward(self): - """Checks that compute_reward returns 0.""" - self.assertEqual(self.env.compute_reward([]), 0) + """Checks that instantiating without get_state implemented + raises an error. + """ + with self.assertRaises(TypeError): + TestFailGetStateEnv(sim_params=self.sim_params, + env_params=self.env_params, + network=self.network) def test__apply_rl_actions(self): - self.assertRaises(NotImplementedError, self.env._apply_rl_actions, - rl_actions=None) + """Checks that instantiating without _apply_rl_actions + implemented raises an error. + """ + with self.assertRaises(TypeError): + TestFailRLActionsEnv(sim_params=self.sim_params, + env_params=self.env_params, + network=self.network) class TestVehicleColoring(unittest.TestCase): @@ -343,7 +385,7 @@ def test_all(self): # add an RL vehicle to ensure that its color will be distinct vehicles.add("rl", acceleration_controller=(RLController, {}), num_vehicles=1) - _, network = ring_road_exp_setup(vehicles=vehicles) + _, network, _ = ring_road_exp_setup(vehicles=vehicles) env = TestEnv(EnvParams(), SumoParams(), network) env.reset() @@ -421,7 +463,7 @@ class TestClipBoxActions(unittest.TestCase): """ def setUp(self): - env, network = ring_road_exp_setup() + env, network, _ = ring_road_exp_setup() sim_params = SumoParams() env_params = EnvParams() self.env = BoxEnv( @@ -469,13 +511,13 @@ class TestClipTupleActions(unittest.TestCase): """ def setUp(self): - env, scenario = ring_road_exp_setup() + env, network, _ = ring_road_exp_setup() sim_params = SumoParams() env_params = EnvParams() self.env = TupleEnv( sim_params=sim_params, env_params=env_params, - scenario=scenario) + network=network) def tearDown(self): self.env.terminate() diff --git a/tests/fast_tests/test_environments.py b/tests/fast_tests/test_environments.py index 560729c6e..48628c4ec 100644 --- a/tests/fast_tests/test_environments.py +++ b/tests/fast_tests/test_environments.py @@ -18,6 +18,9 @@ TestEnv, BottleneckDesiredVelocityEnv, BottleneckEnv, BottleneckAccelEnv from flow.envs.ring.wave_attenuation import v_eq_max_function from flow.envs.multiagent import MultiAgentHighwayPOEnv +from flow.envs.multiagent import MultiAgentAccelPOEnv +from flow.envs.multiagent import MultiAgentWaveAttenuationPOEnv +from flow.envs.multiagent import MultiAgentMergePOEnv os.environ["TEST_FLAG"] = "True" @@ -944,6 +947,278 @@ def test_reset_inflows(self): env.k.vehicle.get_inflow_rate(250)/expected_inflow, 1, 1) +class TestMultiAgentAccelPOEnv(unittest.TestCase): + """Tests the MultiAgentAccelPOEnv environment in + flow/envs/multiagent/ring/accel.py""" + + def setUp(self): + vehicles = VehicleParams() + vehicles.add("rl", acceleration_controller=(RLController, {}), + num_vehicles=1) + vehicles.add("human", acceleration_controller=(IDMController, {}), + num_vehicles=1) + + self.sim_params = SumoParams() + self.network = RingNetwork( + name="test_ring", + vehicles=vehicles, + net_params=NetParams(additional_params=RING_PARAMS.copy()), + ) + self.env_params = EnvParams( + additional_params={ + 'max_accel': 1, + 'max_decel': 1, + "target_velocity": 25 + } + ) + + def tearDown(self): + self.sim_params = None + self.network = None + self.env_params = None + + def test_additional_env_params(self): + """Ensures that not returning the correct params leads to an error.""" + self.assertTrue( + test_additional_params( + env_class=MultiAgentAccelPOEnv, + sim_params=self.sim_params, + network=self.network, + additional_params={ + "max_accel": 1, + "max_decel": 1, + "target_velocity": 10, + } + ) + ) + + def test_observation_action_space(self): + """Tests the observation and action spaces upon initialization.""" + # create the environment + env = MultiAgentAccelPOEnv( + sim_params=self.sim_params, + network=self.network, + env_params=self.env_params + ) + + # check the observation space + self.assertTrue(test_space( + env.observation_space, + expected_size=6, + expected_min=-5, + expected_max=5 + )) + + # check the action space + self.assertTrue(test_space( + env.action_space, + expected_size=1, expected_min=-1, expected_max=1)) + + env.terminate() + + def test_observed(self): + """Ensures that the observed ids are returning the correct vehicles.""" + self.assertTrue( + test_observed( + env_class=MultiAgentMergePOEnv, + sim_params=self.sim_params, + network=self.network, + env_params=self.env_params, + expected_observed=["human_0"] + ) + ) + + +class TestMultiAgentWaveAttenuationPOEnv(unittest.TestCase): + """Tests the MultiAgentWaveAttenuationPOEnv environment in + flow/envs/multiagent/ring/wave_attenuation.py""" + + def setUp(self): + vehicles = VehicleParams() + vehicles.add("rl", acceleration_controller=(RLController, {}), + num_vehicles=1) + vehicles.add("human", acceleration_controller=(IDMController, {}), + num_vehicles=1) + + self.sim_params = SumoParams() + self.network = RingNetwork( + name="test_ring", + vehicles=vehicles, + net_params=NetParams(additional_params=RING_PARAMS.copy()), + ) + self.env_params = EnvParams( + additional_params={ + 'max_accel': 1, + 'max_decel': 1, + "ring_length": [220, 270] + } + ) + + def tearDown(self): + self.sim_params = None + self.network = None + self.env_params = None + + def test_additional_env_params(self): + """Ensures that not returning the correct params leads to an error.""" + self.assertTrue( + test_additional_params( + env_class=MultiAgentWaveAttenuationPOEnv, + sim_params=self.sim_params, + network=self.network, + additional_params={ + "max_accel": 1, + "max_decel": 1, + "ring_length": [220, 270], + } + ) + ) + + def test_observation_action_space(self): + """Tests the observation and action spaces upon initialization.""" + # create the environment + env = MultiAgentWaveAttenuationPOEnv( + sim_params=self.sim_params, + network=self.network, + env_params=self.env_params + ) + + # check the observation space + self.assertTrue(test_space( + env.observation_space, + expected_size=3, + expected_min=-5, + expected_max=5 + )) + + # check the action space + self.assertTrue(test_space( + env.action_space, + expected_size=1, + expected_min=-1, + expected_max=1 + )) + + env.terminate() + + def test_observed(self): + """Ensures that the observed ids are returning the correct vehicles.""" + self.assertTrue( + test_observed( + env_class=MultiAgentWaveAttenuationPOEnv, + sim_params=self.sim_params, + network=self.network, + env_params=self.env_params, + expected_observed=["human_0"] + ) + ) + + def test_reset(self): + """ + Tests that the reset method creating new ring lengths within the + requested range. + """ + # set a random seed to ensure the network lengths are always the same + # during testing + random.seed(9001) + + # create the environment + env = MultiAgentWaveAttenuationPOEnv( + sim_params=self.sim_params, + network=self.network, + env_params=self.env_params + ) + + # reset the network several times and check its length + self.assertEqual(env.k.network.non_internal_length(), 230) + env.reset() + self.assertEqual(env.k.network.non_internal_length(), 239) + env.reset() + self.assertEqual(env.k.network.non_internal_length(), 256) + + +class TestMultiAgentMergePOEnv(unittest.TestCase): + """Tests the MultiAgentMergePOEnv environment in + flow/envs/multiagent/merge.py""" + + def setUp(self): + vehicles = VehicleParams() + vehicles.add("rl", acceleration_controller=(RLController, {}), + num_vehicles=1) + vehicles.add("human", acceleration_controller=(IDMController, {}), + num_vehicles=1) + + self.sim_params = SumoParams() + self.network = MergeNetwork( + name="test_merge", + vehicles=vehicles, + net_params=NetParams(additional_params=MERGE_PARAMS.copy()), + ) + self.env_params = EnvParams( + additional_params={ + 'max_accel': 1, + 'max_decel': 1, + "target_velocity": 25 + } + ) + + def tearDown(self): + self.sim_params = None + self.network = None + self.env_params = None + + def test_additional_env_params(self): + """Ensures that not returning the correct params leads to an error.""" + self.assertTrue( + test_additional_params( + env_class=MultiAgentMergePOEnv, + sim_params=self.sim_params, + network=self.network, + additional_params={ + "max_accel": 1, + "max_decel": 1, + "target_velocity": 10, + } + ) + ) + + def test_observation_action_space(self): + """Tests the observation and action spaces upon initialization.""" + # create the environment + env = MultiAgentMergePOEnv( + sim_params=self.sim_params, + network=self.network, + env_params=self.env_params + ) + + # check the observation space + self.assertTrue(test_space( + env.observation_space, + expected_size=5, + expected_min=-5, + expected_max=5 + )) + + # check the action space + self.assertTrue(test_space( + env.action_space, + expected_size=1, expected_min=-1, expected_max=1)) + + env.terminate() + + def test_observed(self): + """Ensures that the observed ids are returning the correct vehicles.""" + self.assertTrue( + test_observed( + env_class=MultiAgentMergePOEnv, + sim_params=self.sim_params, + network=self.network, + env_params=self.env_params, + expected_observed=["human_0"] + ) + ) + + class TestMultiAgentHighwayPOEnv(unittest.TestCase): def setUp(self): @@ -961,7 +1236,9 @@ def setUp(self): ) self.env_params = EnvParams( additional_params={ - 'max_accel': 1, 'max_decel': 1, "target_velocity": 25 + 'max_accel': 1, + 'max_decel': 1, + "target_velocity": 25 } ) diff --git a/tests/fast_tests/test_examples.py b/tests/fast_tests/test_examples.py index be9dcc9bf..b5faf6517 100644 --- a/tests/fast_tests/test_examples.py +++ b/tests/fast_tests/test_examples.py @@ -1,180 +1,255 @@ +from copy import deepcopy import os import unittest +import random import ray from ray.tune import run_experiments -from examples.rllib.figure_eight import setup_exps as figure_eight_setup -from examples.rllib.traffic_light_grid import setup_exps as traffic_light_grid_setup -from examples.rllib.stabilizing_highway import setup_exps as highway_setup -from examples.rllib.stabilizing_the_ring import setup_exps as ring_setup -from examples.rllib.velocity_bottleneck import setup_exps as bottleneck_setup -from examples.rllib.multiagent_exps.multiagent_figure_eight \ - import setup_exps as multi_figure_eight_setup -from examples.rllib.multiagent_exps.multiagent_stabilizing_the_ring \ - import setup_exps as multi_ring_setup -from examples.rllib.multiagent_exps.multiagent_traffic_light_grid \ - import setup_exps_PPO as multi_grid_setup -from examples.rllib.multiagent_exps.multiagent_traffic_light_grid \ - import make_flow_params as multi_grid_setup_flow_params -from examples.rllib.multiagent_exps.multiagent_highway import flow_params \ - as multi_highway_flow_params -from examples.rllib.multiagent_exps.multiagent_highway import setup_exps \ - as multi_highway_setup - -from examples.stable_baselines.figure_eight import run_model as run_figure_eight -from examples.stable_baselines.traffic_light_grid import run_model as run_traffic_light_grid -from examples.stable_baselines.stabilizing_highway import run_model as run_stabilizing_highway -from examples.stable_baselines.stabilizing_the_ring import run_model as run_stabilizing_ring -from examples.stable_baselines.velocity_bottleneck import run_model as run_velocity_bottleneck - -from examples.sumo.bay_bridge import bay_bridge_example -from examples.sumo.bay_bridge_toll import bay_bridge_toll_example -from examples.sumo.bottlenecks import bottleneck_example -from examples.sumo.density_exp import run_bottleneck -from examples.sumo.figure_eight import figure_eight_example -from examples.sumo.traffic_light_grid import traffic_light_grid_example -from examples.sumo.highway import highway_example -from examples.sumo.highway_ramps import highway_ramps_example -from examples.sumo.merge import merge_example -from examples.sumo.minicity import minicity_example -from examples.sumo.sugiyama import sugiyama_example +from flow.core.experiment import Experiment + +from examples.exp_configs.rl.singleagent.singleagent_figure_eight import flow_params as singleagent_figure_eight +from examples.exp_configs.rl.singleagent.singleagent_traffic_light_grid import \ + flow_params as singleagent_traffic_light_grid +from examples.exp_configs.rl.singleagent.singleagent_merge import flow_params as singleagent_merge +from examples.exp_configs.rl.singleagent.singleagent_ring import flow_params as singleagent_ring +from examples.exp_configs.rl.singleagent.singleagent_bottleneck import flow_params as singleagent_bottleneck + +from examples.exp_configs.rl.multiagent.adversarial_figure_eight import flow_params as adversarial_figure_eight +from examples.exp_configs.rl.multiagent.multiagent_i210 import flow_params as multiagent_i210 +from examples.exp_configs.rl.multiagent.multiagent_figure_eight import flow_params as multiagent_figure_eight +from examples.exp_configs.rl.multiagent.multiagent_merge import flow_params as multiagent_merge +from examples.exp_configs.rl.multiagent.lord_of_the_rings import \ + flow_params as lord_of_the_rings +from examples.exp_configs.rl.multiagent.multiagent_ring import flow_params as multiagent_ring +from examples.exp_configs.rl.multiagent.multiagent_traffic_light_grid import \ + flow_params as multiagent_traffic_light_grid +from examples.exp_configs.rl.multiagent.multiagent_highway import flow_params as multiagent_highway + +from examples.simulate import parse_args as parse_simulate_args +from examples.train import parse_args as parse_train_args +from examples.train import run_model_stablebaseline as run_stable_baselines_model +from examples.train import setup_exps_rllib as setup_rllib_exps +from examples.train import train_h_baselines + +from examples.exp_configs.non_rl.bay_bridge import flow_params as non_rl_bay_bridge +from examples.exp_configs.non_rl.bay_bridge_toll import flow_params as non_rl_bay_bridge_toll +from examples.exp_configs.non_rl.bottleneck import flow_params as non_rl_bottleneck +from examples.exp_configs.non_rl.figure_eight import flow_params as non_rl_figure_eight +from examples.exp_configs.non_rl.traffic_light_grid import flow_params as non_rl_traffic_light_grid +from examples.exp_configs.non_rl.highway import flow_params as non_rl_highway +from examples.exp_configs.non_rl.highway_ramps import flow_params as non_rl_highway_ramps +from examples.exp_configs.non_rl.merge import flow_params as non_rl_merge +from examples.exp_configs.non_rl.minicity import flow_params as non_rl_minicity +from examples.exp_configs.non_rl.ring import flow_params as non_rl_ring +from examples.exp_configs.non_rl.i210_subnetwork import flow_params as non_rl_i210 +from examples.exp_configs.non_rl.highway_single import flow_params as non_rl_highway_single os.environ['TEST_FLAG'] = 'True' os.environ['KMP_DUPLICATE_LIB_OK'] = 'True' +# This removes the randomness in this test +random.seed(a=10) -class TestSumoExamples(unittest.TestCase): - """Tests the example scripts in examples/sumo. - This is done by running the experiment function within each script for a +class TestNonRLExamples(unittest.TestCase): + """Tests the experiment configurations in examples/exp_configs/non_rl. + + This is done by running an experiment form of each config for a few time steps. Note that, this does not test for any refactoring changes done to the functions within the experiment class. """ - def test_bottleneck(self): - """Verifies that examples/sumo/bottlenecks.py is working.""" - # import the experiment variable from the example - exp = bottleneck_example(20, 5, render=False) + def test_parse_args(self): + """Validate the functionality of the parse_args method in simulate.py.""" + # test the default case + args = parse_simulate_args(["exp_config"]) + + self.assertDictEqual(vars(args), { + 'aimsun': False, + 'exp_config': 'exp_config', + 'gen_emission': False, + 'no_render': False, + 'num_runs': 1 + }) - # run the experiment for a few time steps to ensure it doesn't fail - exp.run(1, 5) + # test the case when optional args are specified + args = parse_simulate_args([ + "exp_config", + '--aimsun', + '--gen_emission', + '--no_render', + '--num_runs', '2' + ]) + + self.assertDictEqual(vars(args), { + 'aimsun': True, + 'exp_config': 'exp_config', + 'gen_emission': True, + 'no_render': True, + 'num_runs': 2 + }) - def test_figure_eight(self): - """Verifies that examples/sumo/figure_eight.py is working.""" - # import the experiment variable from the example - exp = figure_eight_example(render=False) + def test_bottleneck(self): + """Verify that examples/exp_configs/non_rl/bottleneck.py is working.""" + self.run_simulation(non_rl_bottleneck) - # run the experiment for a few time steps to ensure it doesn't fail - exp.run(1, 5) + def test_figure_eight(self): + """Verify that examples/exp_configs/non_rl/figure_eight.py is working.""" + self.run_simulation(non_rl_figure_eight) def test_traffic_light_grid(self): - """Verifies that examples/sumo/traffic_light_grid.py is working.""" - # test the example in the absence of inflows - exp = traffic_light_grid_example(render=False, use_inflows=False) - exp.run(1, 5) - - # test the example in the presence of inflows - exp = traffic_light_grid_example(render=False, use_inflows=True) - exp.run(1, 5) + """Verify that examples/exp_configs/non_rl/traffic_light_grid.py is working.""" + self.run_simulation(non_rl_traffic_light_grid) def test_highway(self): - """Verifies that examples/sumo/highway.py is working.""" + """Verify that examples/exp_configs/non_rl/highway.py is working.""" # import the experiment variable from the example - exp = highway_example(render=False) - - # run the experiment for a few time steps to ensure it doesn't fail - exp.run(1, 5) + self.run_simulation(non_rl_highway) def test_highway_ramps(self): - """Verifies that examples/sumo/highway_ramps.py is working.""" - # import the experiment variable from the example - exp = highway_ramps_example(render=False) - - # run the experiment for a few time steps to ensure it doesn't fail - exp.run(1, 5) + """Verify that examples/exp_configs/non_rl/highway_ramps.py is working.""" + self.run_simulation(non_rl_highway_ramps) def test_merge(self): - """Verifies that examples/sumo/merge.py is working.""" - # import the experiment variable from the example - exp = merge_example(render=False) + """Verify that examples/exp_configs/non_rl/merge.py is working.""" + self.run_simulation(non_rl_merge) - # run the experiment for a few time steps to ensure it doesn't fail - exp.run(1, 5) + def test_ring(self): + """Verify that examples/exp_configs/non_rl/ring.py is working.""" + self.run_simulation(non_rl_ring) - def test_sugiyama(self): - """Verifies that examples/sumo/sugiyama.py is working.""" - # import the experiment variable from the example - exp = sugiyama_example(render=False) + def test_bay_bridge(self): + """Verify that examples/exp_configs/non_rl/bay_bridge.py is working.""" + # test without inflows and traffic lights + self.run_simulation(non_rl_bay_bridge) - # run the experiment for a few time steps to ensure it doesn't fail - exp.run(1, 5) + # test with inflows + # FIXME - def test_bay_bridge(self): - """Verifies that examples/sumo/bay_bridge.py is working.""" - # import the experiment variable from the example - exp = bay_bridge_example(render=False) + # test with traffic lights + # FIXME - # run the experiment for a few time steps to ensure it doesn't fail - exp.run(1, 5) + def test_bay_bridge_toll(self): + """Verify that examples/exp_configs/non_rl/bay_bridge_toll.py is working.""" + self.run_simulation(non_rl_bay_bridge_toll) - # import the experiment variable from the example with inflows - exp = bay_bridge_example(render=False, use_inflows=True) + def test_minicity(self): + """Verify that examples/exp_configs/non_rl/minicity.py is working.""" + self.run_simulation(non_rl_minicity) - # run the experiment for a few time steps to ensure it doesn't fail - exp.run(1, 5) + def test_i210(self): + """Verify that examples/exp_configs/non_rl/i210_subnetwork.py is working.""" + self.run_simulation(non_rl_i210) - # import the experiment variable from the example with traffic lights - exp = bay_bridge_example(render=False, use_traffic_lights=True) + def test_highway_single(self): + """Verify that examples/exp_configs/non_rl/highway_single.py is working.""" + self.run_simulation(non_rl_highway_single) - # run the experiment for a few time steps to ensure it doesn't fail - exp.run(1, 5) + @staticmethod + def run_simulation(flow_params): + # make the horizon small and set render to False + flow_params['sim'].render = False + flow_params['env'].horizon = 5 - def test_bay_bridge_toll(self): - """Verifies that examples/sumo/bay_bridge_toll.py is working.""" - # import the experiment variable from the example - exp = bay_bridge_toll_example(render=False) + # create an experiment object + exp = Experiment(flow_params) - # run the experiment for a few time steps to ensure it doesn't fail - exp.run(1, 5) + # run the experiment for one run + exp.run(1) - def test_minicity(self): - """Verifies that examples/sumo/minicity.py is working.""" - # import the experiment variable from the example - exp = minicity_example(render=False) - # run the experiment for a few time steps to ensure it doesn't fail - exp.run(1, 5) +class TestTrain(unittest.TestCase): + + def test_parse_args(self): + """Tests the parse_args method in train.py.""" + # test the default case + args = parse_train_args(["exp_config"]) - def test_density_exp(self): - """Verifies that examples/sumo/density_exp.py is working.""" - run_bottleneck.remote(100, 1, 10, render=False) + self.assertDictEqual(vars(args), { + 'exp_config': 'exp_config', + 'rl_trainer': 'rllib', + 'num_cpus': 1, + 'num_steps': 5000, + 'rollout_size': 1000, + 'checkpoint_path': None + }) + + # test the case when optional args are specified + args = parse_train_args([ + "exp_config", + "--rl_trainer", "h-baselines", + "--num_cpus" "2", + "--num_steps", "3", + "--rollout_size", "4", + "--checkpoint_path", "5", + ]) + + self.assertDictEqual(vars(args), { + 'checkpoint_path': '5', + 'exp_config': 'exp_config', + 'num_cpus': 1, + 'num_steps': 3, + 'rl_trainer': 'h-baselines', + 'rollout_size': 4 + }) class TestStableBaselineExamples(unittest.TestCase): - """Tests the example scripts in examples/stable_baselines. + """Tests the example scripts in examples/exp_configs/rl/singleagent for stable_baselines. - This is done by running each experiment in that folder for five time-steps - and confirming that it completes one rollout with two workers. + This is done by running each experiment in that folder for five time-steps + and confirming that it completes one rollout with two workers. """ - def test_run_traffic_light_grid(self): - run_traffic_light_grid(num_steps=5) + @staticmethod + def run_exp(flow_params): + train_model = run_stable_baselines_model(flow_params, 1, 4, 4) + train_model.env.close() + + def test_singleagent_figure_eight(self): + self.run_exp(singleagent_figure_eight) - def test_run_figure_eight(self): - run_figure_eight(num_steps=5) + def test_singleagent_traffic_light_grid(self): + self.run_exp(singleagent_traffic_light_grid) - def test_run_stabilizing_highway(self): - run_stabilizing_highway(num_steps=5) + def test_singleagent_merge(self): + self.run_exp(singleagent_merge) + + def test_singleagent_ring(self): + self.run_exp(singleagent_ring) + + def test_singleagent_bottleneck(self): + self.run_exp(singleagent_bottleneck) + + +class TestHBaselineExamples(unittest.TestCase): + """Tests the functionality of the h-baselines features in train.py. + + This is done by running a set of experiments for 10 time-steps and + confirming that it runs. + """ + @staticmethod + def run_exp(env_name, multiagent): + train_h_baselines( + env_name=env_name, + args=[ + env_name, + "--initial_exploration_steps", "1", + "--total_steps", "10" + ], + multiagent=multiagent, + ) - def test_run_stabilizing_ring(self): - run_stabilizing_ring(num_steps=5) + def test_singleagent_ring(self): + self.run_exp("singleagent_ring", multiagent=False) - def test_run_velocity_bottleneck(self): - run_velocity_bottleneck(num_steps=5) + def test_multiagent_ring(self): + self.run_exp("multiagent_ring", multiagent=True) class TestRllibExamples(unittest.TestCase): - """Tests the example scripts in examples/rllib. + """Tests the example scripts in examples/exp_configs/rl/singleagent and + examples/exp_configs/rl/multiagent for RLlib. This is done by running each experiment in that folder for five time-steps and confirming that it completes one rollout with two workers. @@ -184,52 +259,155 @@ def setUp(self): if not ray.is_initialized(): ray.init(num_cpus=1) - def test_figure_eight(self): - alg_run, env_name, config = figure_eight_setup() - self.run_exp(alg_run, env_name, config) - - def test_traffic_light_grid(self): - # test the example in the absence of inflows - alg_run, env_name, config = traffic_light_grid_setup(use_inflows=False) - self.run_exp(alg_run, env_name, config) - - def test_traffic_light_grid_inflows(self): - # test the example in the presence of inflows - alg_run, env_name, config = traffic_light_grid_setup(use_inflows=True) - self.run_exp(alg_run, env_name, config) - - def test_stabilizing_highway(self): - alg_run, env_name, config = highway_setup() - self.run_exp(alg_run, env_name, config) - - def test_ring(self): - alg_run, env_name, config = ring_setup() - self.run_exp(alg_run, env_name, config) - - def test_bottleneck(self): - alg_run, env_name, config = bottleneck_setup() - self.run_exp(alg_run, env_name, config) - - def test_multi_figure_eight(self): - alg_run, env_name, config = multi_figure_eight_setup() - self.run_exp(alg_run, env_name, config) - - def test_multi_ring(self): - alg_run, env_name, config = multi_ring_setup() - self.run_exp(alg_run, env_name, config) - - def test_multi_grid(self): - flow_params = multi_grid_setup_flow_params(1, 1, 300) - alg_run, env_name, config = multi_grid_setup(flow_params) - self.run_exp(alg_run, env_name, config) + def test_singleagent_figure_eight(self): + self.run_exp(singleagent_figure_eight) + + def test_singleagent_traffic_light_grid(self): + self.run_exp(singleagent_traffic_light_grid) + + def test_singleagent_traffic_light_grid_inflows(self): + pass # FIXME + + def test_singleagent_merge(self): + self.run_exp(singleagent_merge) + + def test_singleagent_ring(self): + self.run_exp(singleagent_ring) + + def test_singleagent_bottleneck(self): + self.run_exp(singleagent_bottleneck) + + def test_adversarial_figure_eight(self): + from examples.exp_configs.rl.multiagent.adversarial_figure_eight import POLICY_GRAPHS as af8pg + from examples.exp_configs.rl.multiagent.adversarial_figure_eight import policy_mapping_fn as af8pmf + + kwargs = { + "policy_graphs": af8pg, + "policy_mapping_fn": af8pmf + } + self.run_exp(adversarial_figure_eight, **kwargs) + + def test_multiagent_figure_eight(self): + from examples.exp_configs.rl.multiagent.multiagent_figure_eight import POLICY_GRAPHS as mf8pg + from examples.exp_configs.rl.multiagent.multiagent_figure_eight import policy_mapping_fn as mf8pmf + + kwargs = { + "policy_graphs": mf8pg, + "policy_mapping_fn": mf8pmf + } + self.run_exp(multiagent_figure_eight, **kwargs) + + def test_lord_of_the_rings(self): + from examples.exp_configs.rl.multiagent.lord_of_the_rings import POLICY_GRAPHS as ltrpg + from examples.exp_configs.rl.multiagent.lord_of_the_rings import POLICIES_TO_TRAIN as ltrpt + from examples.exp_configs.rl.multiagent.lord_of_the_rings import policy_mapping_fn as ltrpmf + + kwargs = { + "policy_graphs": ltrpg, + "policies_to_train": ltrpt, + "policy_mapping_fn": ltrpmf + } + self.run_exp(lord_of_the_rings, **kwargs) + + def test_multiagent_ring(self): + from examples.exp_configs.rl.multiagent.multiagent_ring import POLICY_GRAPHS as mrpg + from examples.exp_configs.rl.multiagent.multiagent_ring import policy_mapping_fn as mrpmf + + kwargs = { + "policy_graphs": mrpg, + "policy_mapping_fn": mrpmf + } + self.run_exp(multiagent_ring, **kwargs) + + def test_multiagent_merge(self): + from examples.exp_configs.rl.multiagent.multiagent_merge import POLICY_GRAPHS as mmpg + from examples.exp_configs.rl.multiagent.multiagent_merge import policy_mapping_fn as mmpmf + + kwargs = { + "policy_graphs": mmpg, + "policy_mapping_fn": mmpmf + } + self.run_exp(multiagent_merge, **kwargs) + + def test_multi_traffic_light_grid(self): + from examples.exp_configs.rl.multiagent.multiagent_traffic_light_grid import POLICY_GRAPHS as mtlpg + from examples.exp_configs.rl.multiagent.multiagent_traffic_light_grid import POLICIES_TO_TRAIN as mtlpt + from examples.exp_configs.rl.multiagent.multiagent_traffic_light_grid import policy_mapping_fn as mtlpmf + + kwargs = { + "policy_graphs": mtlpg, + "policies_to_train": mtlpt, + "policy_mapping_fn": mtlpmf + } + self.run_exp(multiagent_traffic_light_grid, **kwargs) def test_multi_highway(self): - flow_params = multi_highway_flow_params - alg_run, env_name, config = multi_highway_setup(flow_params) - self.run_exp(alg_run, env_name, config) + from examples.exp_configs.rl.multiagent.multiagent_highway import POLICY_GRAPHS as mhpg + from examples.exp_configs.rl.multiagent.multiagent_highway import POLICIES_TO_TRAIN as mhpt + from examples.exp_configs.rl.multiagent.multiagent_highway import policy_mapping_fn as mhpmf + + kwargs = { + "policy_graphs": mhpg, + "policies_to_train": mhpt, + "policy_mapping_fn": mhpmf + } + self.run_exp(multiagent_highway, **kwargs) + + def test_multiagent_i210(self): + from examples.exp_configs.rl.multiagent.multiagent_i210 import POLICIES_TO_TRAIN as mi210pr + from examples.exp_configs.rl.multiagent.multiagent_i210 import policy_mapping_fn as mi210mf + + from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy + from ray.tune.registry import register_env + from flow.utils.registry import make_create_env + # test observation space 1 + flow_params = deepcopy(multiagent_i210) + flow_params['env'].additional_params['lead_obs'] = True + create_env, env_name = make_create_env(params=flow_params, version=0) + + # register as rllib env + register_env(env_name, create_env) + + # multiagent configuration + test_env = create_env() + obs_space = test_env.observation_space + act_space = test_env.action_space + + POLICY_GRAPHS = {'av': (PPOTFPolicy, obs_space, act_space, {})} + + kwargs = { + "policy_graphs": POLICY_GRAPHS, + "policies_to_train": mi210pr, + "policy_mapping_fn": mi210mf + } + self.run_exp(flow_params, **kwargs) + + # test observation space 2 + flow_params = deepcopy(multiagent_i210) + flow_params['env'].additional_params['lead_obs'] = False + create_env, env_name = make_create_env(params=flow_params, version=0) + + # register as rllib env + register_env(env_name, create_env) + + # multiagent configuration + test_env = create_env() + obs_space = test_env.observation_space + act_space = test_env.action_space + + POLICY_GRAPHS = {'av': (PPOTFPolicy, obs_space, act_space, {})} + + kwargs = { + "policy_graphs": POLICY_GRAPHS, + "policies_to_train": mi210pr, + "policy_mapping_fn": mi210mf + } + self.run_exp(flow_params, **kwargs) @staticmethod - def run_exp(alg_run, env_name, config): + def run_exp(flow_params, **kwargs): + alg_run, env_name, config = setup_rllib_exps(flow_params, 1, 1, **kwargs) + try: ray.init(num_cpus=1) except Exception as e: diff --git a/tests/fast_tests/test_experiment_base_class.py b/tests/fast_tests/test_experiment_base_class.py index c71dc890b..458af1027 100644 --- a/tests/fast_tests/test_experiment_base_class.py +++ b/tests/fast_tests/test_experiment_base_class.py @@ -1,14 +1,20 @@ import unittest import os import time +import csv from flow.core.experiment import Experiment from flow.core.params import VehicleParams -from flow.controllers import RLController, ContinuousRouter +from flow.controllers import IDMController, RLController, ContinuousRouter from flow.core.params import SumoCarFollowingParams from flow.core.params import SumoParams +from flow.core.params import EnvParams, InitialConfig, NetParams +from flow.core.params import TrafficLightParams +from flow.envs import AccelEnv +from flow.networks import RingNetwork from tests.setup_scripts import ring_road_exp_setup + import numpy as np os.environ["TEST_FLAG"] = "True" @@ -21,17 +27,19 @@ class TestNumSteps(unittest.TestCase): def setUp(self): # create the environment and network classes for a ring road - env, _ = ring_road_exp_setup() - + env, _, flow_params = ring_road_exp_setup() + flow_params['sim'].render = False + flow_params['env'].horizon = 10 # instantiate an experiment class - self.exp = Experiment(env) + self.exp = Experiment(flow_params) + self.exp.env = env def tearDown(self): # free up used memory self.exp = None def test_steps(self): - self.exp.run(num_runs=1, num_steps=10) + self.exp.run(num_runs=1) self.assertEqual(self.exp.env.time_counter, 10) @@ -45,17 +53,24 @@ class TestNumRuns(unittest.TestCase): def test_num_runs(self): # run the experiment for 1 run and collect the last position of all # vehicles - env, _ = ring_road_exp_setup() - exp = Experiment(env) - exp.run(num_runs=1, num_steps=10) + env, _, flow_params = ring_road_exp_setup() + flow_params['sim'].render = False + flow_params['env'].horizon = 10 + exp = Experiment(flow_params) + exp.env = env + exp.run(num_runs=1) vel1 = [exp.env.k.vehicle.get_speed(exp.env.k.vehicle.get_ids())] # run the experiment for 2 runs and collect the last position of all # vehicles - env, _ = ring_road_exp_setup() - exp = Experiment(env) - exp.run(num_runs=2, num_steps=10) + env, _, flow_params = ring_road_exp_setup() + flow_params['sim'].render = False + flow_params['env'].horizon = 10 + + exp = Experiment(flow_params) + exp.env = env + exp.run(num_runs=2) vel2 = [exp.env.k.vehicle.get_speed(exp.env.k.vehicle.get_ids())] @@ -84,11 +99,12 @@ def rl_actions(*_): ), num_vehicles=1) - env, _ = ring_road_exp_setup(vehicles=vehicles) - - exp = Experiment(env=env) - - exp.run(1, 10, rl_actions=rl_actions) + env, _, flow_params = ring_road_exp_setup(vehicles=vehicles) + flow_params['sim'].render = False + flow_params['env'].horizon = 10 + exp = Experiment(flow_params) + exp.env = env + exp.run(1, rl_actions=rl_actions) # check that the acceleration of the RL vehicle was that specified by # the rl_actions method @@ -105,23 +121,93 @@ class TestConvertToCSV(unittest.TestCase): def test_convert_to_csv(self): dir_path = os.path.dirname(os.path.realpath(__file__)) sim_params = SumoParams(emission_path="{}/".format(dir_path)) - env, network = ring_road_exp_setup(sim_params=sim_params) - exp = Experiment(env) - exp.run(num_runs=1, num_steps=10, convert_to_csv=True) - time.sleep(0.1) + vehicles = VehicleParams() + vehicles.add( + veh_id="idm", + acceleration_controller=(IDMController, {}), + routing_controller=(ContinuousRouter, {}), + car_following_params=SumoCarFollowingParams( + speed_mode="aggressive", + ), + num_vehicles=1) + + additional_env_params = { + "target_velocity": 8, + "max_accel": 1, + "max_decel": 1, + "sort_vehicles": False, + } + env_params = EnvParams( + horizon=10, + additional_params=additional_env_params) + + additional_net_params = { + "length": 230, + "lanes": 1, + "speed_limit": 30, + "resolution": 40 + } + net_params = NetParams(additional_params=additional_net_params) + + flow_params = dict( + exp_tag="RingRoadTest", + env_name=AccelEnv, + network=RingNetwork, + simulator='traci', + sim=sim_params, + env=env_params, + net=net_params, + veh=vehicles, + initial=InitialConfig(lanes_distribution=1), + tls=TrafficLightParams(), + ) + + exp = Experiment(flow_params) + exp.run(num_runs=1, convert_to_csv=True) + + time.sleep(1.0) # check that both the csv file exists and the xml file doesn't. - self.assertFalse(os.path.isfile(dir_path + "/{}-emission.xml".format( - network.name))) - self.assertTrue(os.path.isfile(dir_path + "/{}-emission.csv".format( - network.name))) + self.assertFalse(os.path.isfile(dir_path + "/{}-0_emission.xml".format( + exp.env.network.name))) + self.assertTrue(os.path.isfile(dir_path + "/{}-0_emission.csv".format( + exp.env.network.name))) + + # check that the keys within the emission file matches its expected + # values + with open(dir_path + "/{}-0_emission.csv".format( + exp.env.network.name), "r") as f: + reader = csv.reader(f) + header = next(reader) + + self.assertListEqual(header, [ + "time", + "id", + "x", + "y", + "speed", + "headway", + "leader_id", + "target_accel_with_noise_with_failsafe", + "target_accel_no_noise_no_failsafe", + "target_accel_with_noise_no_failsafe", + "target_accel_no_noise_with_failsafe", + "realized_accel", + "road_grade", + "edge_id", + "lane_number", + "distance", + "relative_position", + "follower_id", + "leader_rel_speed", + ]) time.sleep(0.1) # delete the files - os.remove(os.path.expanduser(dir_path + "/{}-emission.csv".format( - network.name))) + os.remove(os.path.expanduser(dir_path + "/{}-0_emission.csv".format( + exp.env.network.name))) if __name__ == '__main__': diff --git a/tests/fast_tests/test_files/i210_emission.csv b/tests/fast_tests/test_files/i210_emission.csv new file mode 100644 index 000000000..ec63cf9cf --- /dev/null +++ b/tests/fast_tests/test_files/i210_emission.csv @@ -0,0 +1,27 @@ +x,time,edge_id,eclass,type,PMx,speed,angle,CO,CO2,electricity,noise,lane_number,NOx,distance,route,y,id,fuel,HC,waiting +485.04,0.8,119257914,HBEFA3/PC_G_EU4,human,0.05,23.0,119.74,3.32,3793.12,0.0,70.29,1,1.17,5.1,route119257914_0,1068.18,flow_00.0,1.63,0.11,0.0 +500.91,1.6,119257914,HBEFA3/PC_G_EU4,human,0.0,22.84,119.74,0.0,0.0,0.0,69.9,1,0.0,23.37,route119257914_0,1059.12,flow_00.0,0.0,0.0,0.0 +517.1,2.4,119257914,HBEFA3/PC_G_EU4,human,0.15,23.31,119.74,78.83,7435.5,0.0,71.61,1,2.88,42.02,route119257914_0,1049.87,flow_00.0,3.2,0.54,0.0 +533.76,3.2,119257914,HBEFA3/PC_G_EU4,human,0.2,23.98,119.74,117.59,9250.57,0.0,72.56,1,3.72,61.21,route119257914_0,1040.35,flow_00.0,3.98,0.76,0.0 +550.83,4.0,119257908#0,HBEFA3/PC_G_EU4,human,0.12,24.25,118.82,55.17,6186.6,0.0,71.56,1,2.27,18.87,route119257914_0,1030.94,flow_00.0,2.66,0.4,0.0 +569.33,4.8,119257908#0,HBEFA3/PC_G_EU4,human,0.55,26.33,118.82,398.05,22491.67,0.0,79.56,1,9.92,39.93,route119257914_0,1020.77,flow_00.0,9.67,2.37,0.0 +488.22,0.8,119257914,HBEFA3/PC_G_EU4,human,0.05,23.0,119.74,3.32,3793.12,0.0,70.29,3,1.17,5.1,route119257914_0,1073.74,flow_00.1,1.63,0.11,0.0 +504.38,1.6,119257914,HBEFA3/PC_G_EU4,human,0.11,23.27,119.74,47.84,5933.43,0.0,71.06,3,2.18,23.72,route119257914_0,1064.51,flow_00.1,2.55,0.36,0.0 +521.17,2.4,119257914,HBEFA3/PC_G_EU4,human,0.25,24.18,119.74,158.54,11215.9,0.0,73.46,3,4.65,43.06,route119257914_0,1054.91,flow_00.1,4.82,1.0,0.0 +538.52,3.2,119257908#0,HBEFA3/PC_G_EU4,human,0.25,25.01,119.48,158.26,11085.45,0.0,73.63,3,4.57,1.33,route119257914_0,1045.03,flow_00.1,4.77,1.0,0.0 +556.27,4.0,119257908#0,HBEFA3/PC_G_EU4,human,0.16,25.39,118.82,86.12,7519.43,0.0,72.45,3,2.88,21.65,route119257914_0,1035.26,flow_00.1,3.23,0.58,0.0 +575.34,4.8,119257908#0,HBEFA3/PC_G_EU4,human,0.53,27.27,118.82,383.84,21586.2,0.0,79.03,3,9.47,43.46,route119257914_0,1024.77,flow_00.1,9.28,2.29,0.0 +486.63,2.4,119257914,HBEFA3/PC_G_EU4,human,0.05,23.0,119.74,3.32,3793.12,0.0,70.29,2,1.17,5.1,route119257914_0,1070.96,flow_00.2,1.63,0.11,0.0 +503.03,3.2,119257914,HBEFA3/PC_G_EU4,human,0.18,23.61,119.74,103.29,8596.1,0.0,72.17,2,3.42,23.98,route119257914_0,1061.59,flow_00.2,3.7,0.68,0.0 +519.7,4.0,119257914,HBEFA3/PC_G_EU4,human,0.14,24.0,119.74,73.27,7095.73,0.0,71.76,2,2.71,43.18,route119257914_0,1052.07,flow_00.2,3.05,0.51,0.0 +489.8,2.4,119257914,HBEFA3/PC_G_EU4,human,0.05,23.0,119.74,3.32,3793.12,0.0,70.29,4,1.17,5.1,route119257914_0,1076.52,flow_00.3,1.63,0.11,0.0 +505.86,3.2,119257914,HBEFA3/PC_G_EU4,human,0.08,23.12,119.74,23.4,4758.73,0.0,70.63,4,1.62,23.6,route119257914_0,1067.34,flow_00.3,2.05,0.22,0.0 +522.24,4.0,119257914,HBEFA3/PC_G_EU4,human,0.15,23.58,119.74,78.96,7417.2,0.0,71.71,4,2.87,42.46,route119257914_0,1057.98,flow_00.3,3.19,0.54,0.0 +488.22,2.4,119257914,HBEFA3/PC_G_EU4,human,0.05,23.0,119.74,3.32,3793.12,0.0,70.29,3,1.17,5.1,route119257914_0,1073.74,flow_00.4,1.63,0.11,0.0 +504.4,3.2,119257914,HBEFA3/PC_G_EU4,human,0.11,23.3,119.74,51.58,6113.11,0.0,71.13,3,2.26,23.74,route119257914_0,1064.5,flow_00.4,2.63,0.39,0.0 +520.58,4.0,119257914,HBEFA3/PC_G_EU4,human,0.06,23.31,119.74,6.86,3940.44,0.0,70.48,3,1.23,42.38,route119257914_0,1055.25,flow_00.4,1.69,0.13,0.0 +485.04,2.4,119257914,HBEFA3/PC_G_EU4,human,0.05,23.0,119.74,3.32,3793.12,0.0,70.29,1,1.17,5.1,route119257914_0,1068.18,flow_00.5,1.63,0.11,0.0 +500.55,3.2,119257914,HBEFA3/PC_G_EU4,human,0.0,22.33,119.74,0.0,0.0,0.0,68.86,1,0.0,22.97,route119257914_0,1059.32,flow_00.5,0.0,0.0,0.0 +515.98,4.0,119257914,HBEFA3/PC_G_EU4,human,0.0,22.21,119.74,0.0,0.0,0.0,69.59,1,0.0,40.73,route119257914_0,1050.51,flow_00.5,0.0,0.0,0.0 +488.22,4.0,119257914,HBEFA3/PC_G_EU4,human,0.05,23.0,119.74,3.32,3793.12,0.0,70.29,3,1.17,5.1,route119257914_0,1073.74,flow_00.6,1.63,0.11,0.0 +485.04,4.0,119257914,HBEFA3/PC_G_EU4,human,0.05,23.0,119.74,3.32,3793.12,0.0,70.29,1,1.17,5.1,route119257914_0,1068.18,flow_00.7,1.63,0.11,0.0 diff --git a/tests/fast_tests/test_files/ring_230.json b/tests/fast_tests/test_files/ring_230.json index 68be938aa..12b0ec61d 100644 --- a/tests/fast_tests/test_files/ring_230.json +++ b/tests/fast_tests/test_files/ring_230.json @@ -15,7 +15,7 @@ "warmup_steps": 750 }, "env_name": "WaveAttenuationPOEnv", - "exp_tag": "stabilizing_the_ring", + "exp_tag": "singleagent_ring", "initial": { "additional_params": {}, "bunching": 0, diff --git a/tests/fast_tests/test_params.py b/tests/fast_tests/test_params.py index d009af489..df4794d28 100644 --- a/tests/fast_tests/test_params.py +++ b/tests/fast_tests/test_params.py @@ -203,8 +203,7 @@ def test_lc_params(self): expected_attributes_2 = \ ["laneChangeModel", "lcStrategic", "lcCooperative", "lcSpeedGain", "lcKeepRight", "lcLookaheadLeft", "lcSpeedGainRight", "lcSublane", - "lcPushy", "lcPushyGap", "lcAssertive", "lcImpatience", - "lcTimeToImpatience", "lcAccelLat"] + "lcPushy", "lcPushyGap", "lcAssertive", "lcAccelLat"] self.assertCountEqual(attributes_2, expected_attributes_2) def test_wrong_model(self): @@ -240,9 +239,7 @@ def test_deprecated(self): lcSublane=1.0, lcPushy=0, lcPushyGap=0.6, - lcAssertive=1, - lcImpatience=0, - lcTimeToImpatience=float("inf")) + lcAssertive=1) # ensure that the attributes match their correct element in the # "controller_params" dict @@ -262,8 +259,6 @@ def test_deprecated(self): float(lc_params.controller_params["lcPushyGap"]), 0.6) self.assertAlmostEqual( float(lc_params.controller_params["lcAssertive"]), 1) - self.assertAlmostEqual( - float(lc_params.controller_params["lcImpatience"]), 0) if __name__ == '__main__': diff --git a/tests/fast_tests/test_rewards.py b/tests/fast_tests/test_rewards.py index be616e0bc..3f2e08cde 100644 --- a/tests/fast_tests/test_rewards.py +++ b/tests/fast_tests/test_rewards.py @@ -7,6 +7,7 @@ from flow.core.rewards import average_velocity, min_delay from flow.core.rewards import desired_velocity, boolean_action_penalty from flow.core.rewards import penalize_near_standstill, penalize_standstill +from flow.core.rewards import energy_consumption os.environ["TEST_FLAG"] = "True" @@ -23,8 +24,8 @@ def test_desired_velocity(self): "target_velocity": np.sqrt(10), "max_accel": 1, "max_decel": 1, "sort_vehicles": False}) - env, _ = ring_road_exp_setup(vehicles=vehicles, - env_params=env_params) + env, _, _ = ring_road_exp_setup(vehicles=vehicles, + env_params=env_params) # check that the fail attribute leads to a zero return self.assertEqual(desired_velocity(env, fail=True), 0) @@ -61,7 +62,7 @@ def test_average_velocity(self): vehicles = VehicleParams() vehicles.add("test", num_vehicles=10) - env, _ = ring_road_exp_setup(vehicles=vehicles) + env, _, _ = ring_road_exp_setup(vehicles=vehicles) # check that the fail attribute leads to a zero return self.assertEqual(average_velocity(env, fail=True), 0) @@ -77,7 +78,7 @@ def test_average_velocity(self): # recreate the environment with no vehicles vehicles = VehicleParams() - env, _ = ring_road_exp_setup(vehicles=vehicles) + env, _, _ = ring_road_exp_setup(vehicles=vehicles) # check that the reward function return 0 in the case of no vehicles self.assertEqual(average_velocity(env, fail=False), 0) @@ -86,7 +87,7 @@ def test_min_delay(self): """Test the min_delay method.""" # try the case of an environment with no vehicles vehicles = VehicleParams() - env, _ = ring_road_exp_setup(vehicles=vehicles) + env, _, _ = ring_road_exp_setup(vehicles=vehicles) # check that the reward function return 0 in the case of no vehicles self.assertEqual(min_delay(env), 0) @@ -94,7 +95,7 @@ def test_min_delay(self): # try the case of multiple vehicles vehicles = VehicleParams() vehicles.add("test", num_vehicles=10) - env, _ = ring_road_exp_setup(vehicles=vehicles) + env, _, _ = ring_road_exp_setup(vehicles=vehicles) # check the min_delay upon reset self.assertAlmostEqual(min_delay(env), 0) @@ -114,8 +115,8 @@ def test_penalize_standstill(self): "target_velocity": 10, "max_accel": 1, "max_decel": 1, "sort_vehicles": False}) - env, _ = ring_road_exp_setup(vehicles=vehicles, - env_params=env_params) + env, _, _ = ring_road_exp_setup(vehicles=vehicles, + env_params=env_params) # check the penalty is acknowledging all vehicles self.assertEqual(penalize_standstill(env, gain=1), -10) @@ -137,8 +138,8 @@ def test_penalize_near_standstill(self): "target_velocity": 10, "max_accel": 1, "max_decel": 1, "sort_vehicles": False}) - env, _ = ring_road_exp_setup(vehicles=vehicles, - env_params=env_params) + env, _, _ = ring_road_exp_setup(vehicles=vehicles, + env_params=env_params) # check the penalty is acknowledging all vehicles self.assertEqual(penalize_near_standstill(env, gain=1), -10) @@ -151,6 +152,31 @@ def test_penalize_near_standstill(self): self.assertEqual(penalize_near_standstill(env, thresh=2), -10) self.assertEqual(penalize_near_standstill(env, thresh=0.5), -9) + def test_energy_consumption(self): + """Test the energy consumption method.""" + vehicles = VehicleParams() + vehicles.add("test", num_vehicles=10) + + env_params = EnvParams(additional_params={ + "target_velocity": 10, "max_accel": 1, "max_decel": 1, + "sort_vehicles": False}) + + env, _, _ = ring_road_exp_setup(vehicles=vehicles, + env_params=env_params) + + # check the penalty is zero at speed zero + self.assertEqual(energy_consumption(env, gain=1), 0) + + # change the speed of one vehicle + env.k.vehicle.test_set_speed("test_0", 1) + self.assertEqual(energy_consumption(env), -12.059337750000001) + + # check that stepping change the previous speeds and increases the energy consumption + env.step(rl_actions=None) + env.step(rl_actions=None) + self.assertGreater(env.k.vehicle.get_previous_speed("test_0"), 0.0) + self.assertLess(energy_consumption(env), -12.059337750000001) + def test_boolean_action_penalty(self): """Test the boolean_action_penalty method.""" actions = [False, False, False, False, False] diff --git a/tests/fast_tests/test_scenario_base_class.py b/tests/fast_tests/test_scenario_base_class.py index 18f449468..e563c07ab 100644 --- a/tests/fast_tests/test_scenario_base_class.py +++ b/tests/fast_tests/test_scenario_base_class.py @@ -42,7 +42,7 @@ class TestGetX(unittest.TestCase): def setUp(self): # create the environment and network classes for a figure eight - self.env, _ = figure_eight_exp_setup() + self.env, _, _ = figure_eight_exp_setup() def tearDown(self): # free data used by the class @@ -74,7 +74,7 @@ class TestGetEdge(unittest.TestCase): def setUp(self): # create the environment and network classes for a figure eight - self.env, _ = figure_eight_exp_setup() + self.env, _, _ = figure_eight_exp_setup() def tearDown(self): # free data used by the class @@ -128,7 +128,7 @@ def setUp_gen_start_pos(self, initial_config=InitialConfig()): num_vehicles=15) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup( + self.env, _, _ = ring_road_exp_setup( net_params=net_params, initial_config=initial_config, vehicles=vehicles) @@ -453,7 +453,7 @@ def setUp(self): initial_config = InitialConfig(x0=150) # create the environment and network classes for a ring road - self.env, _ = figure_eight_exp_setup( + self.env, _, _ = figure_eight_exp_setup( initial_config=initial_config, vehicles=vehicles) def tearDown(self): @@ -527,7 +527,7 @@ def setUp_gen_start_pos(self, initial_config=InitialConfig()): num_vehicles=5) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup( + self.env, _, _ = ring_road_exp_setup( net_params=net_params, initial_config=initial_config, vehicles=vehicles) @@ -609,7 +609,7 @@ def setUp(self): # create the environment and network classes for a variable lanes per # edge ring road - self.env, _ = variable_lanes_exp_setup( + self.env, _, _ = variable_lanes_exp_setup( vehicles=vehicles, initial_config=initial_config) def tearDown(self): @@ -644,7 +644,7 @@ def setUp(self): # create the environment and network classes for a variable lanes per # edge ring road - self.env, _ = variable_lanes_exp_setup( + self.env, _, _ = variable_lanes_exp_setup( vehicles=vehicles, initial_config=initial_config) @@ -666,7 +666,7 @@ def test_edge_length_edges(self): net_params = NetParams(additional_params=additional_net_params) # create the environment and network classes for a ring road - env, _ = ring_road_exp_setup(net_params=net_params) + env, _, _ = ring_road_exp_setup(net_params=net_params) self.assertEqual(env.k.network.edge_length("top"), 250) @@ -685,7 +685,7 @@ def test_edge_length_junctions(self): } net_params = NetParams(additional_params=additional_net_params) - env, _ = figure_eight_exp_setup(net_params=net_params) + env, _, _ = figure_eight_exp_setup(net_params=net_params) self.assertAlmostEqual( env.k.network.edge_length(":center_0"), 9.40) # FIXME: 6.2? @@ -711,7 +711,7 @@ def test_speed_limit_edges(self): net_params = NetParams(additional_params=additional_net_params) # create the environment and network classes for a figure eight - env, _ = ring_road_exp_setup(net_params=net_params) + env, _, _ = ring_road_exp_setup(net_params=net_params) self.assertAlmostEqual(env.k.network.speed_limit("top"), 60) @@ -730,7 +730,7 @@ def test_speed_limit_junctions(self): } net_params = NetParams(additional_params=additional_net_params) - env, network = figure_eight_exp_setup(net_params=net_params) + env, network, _ = figure_eight_exp_setup(net_params=net_params) self.assertAlmostEqual( env.k.network.speed_limit("bottom"), 60) @@ -756,7 +756,7 @@ def test_num_lanes_edges(self): net_params = NetParams(additional_params=additional_net_params) # create the environment and network classes for a figure eight - env, network = ring_road_exp_setup(net_params=net_params) + env, network, _ = ring_road_exp_setup(net_params=net_params) self.assertEqual(env.k.network.num_lanes("top"), 2) @@ -775,7 +775,7 @@ def test_num_lanes_junctions(self): } net_params = NetParams(additional_params=additional_net_params) - env, network = figure_eight_exp_setup(net_params=net_params) + env, network, _ = figure_eight_exp_setup(net_params=net_params) self.assertEqual(env.k.network.num_lanes("bottom"), 3) self.assertEqual(env.k.network.num_lanes(":top_0"), 3) @@ -789,7 +789,7 @@ class TestGetEdgeList(unittest.TestCase): def setUp(self): # create the environment and network classes for a figure eight - self.env, _ = figure_eight_exp_setup() + self.env, _, _ = figure_eight_exp_setup() def tearDown(self): # free data used by the class @@ -812,7 +812,7 @@ class TestGetJunctionList(unittest.TestCase): def setUp(self): # create the environment and network classes for a figure eight - self.env, _ = figure_eight_exp_setup() + self.env, _, _ = figure_eight_exp_setup() def tearDown(self): # free data used by the class @@ -840,7 +840,7 @@ def test_next_prev_edge_figure_eight(self): """ Tests the next_edge() and prev_edge() methods for the figure eight. """ - env, _ = figure_eight_exp_setup() + env, _, _ = figure_eight_exp_setup() next_edge = env.k.network.next_edge("bottom", 0) expected_next_edge = [(':center_1', 0)] @@ -854,7 +854,7 @@ def test_next_prev_edge_ring_road(self): """ Tests the next_edge() and prev_edge() methods for the ring road. """ - env, _ = ring_road_exp_setup() + env, _, _ = ring_road_exp_setup() next_edge = env.k.network.next_edge("top", 0) expected_next_edge = [(":left_0", 0)] @@ -869,7 +869,7 @@ def test_no_edge_ahead(self): Tests that, when there are no edges in front, next_edge() returns an empty list """ - env, _ = highway_exp_setup() + env, _, _ = highway_exp_setup() next_edge = env.k.network.next_edge( env.k.network.get_edge_list()[0], 0) self.assertTrue(len(next_edge) == 0) @@ -879,7 +879,7 @@ def test_no_edge_behind(self): Tests that, when there are no edges behind, prev_edge() returns an empty list """ - env, _ = highway_exp_setup() + env, _, _ = highway_exp_setup() prev_edge = env.k.network.prev_edge( env.k.network.get_edge_list()[0], 0) self.assertTrue(len(prev_edge) == 0) diff --git a/tests/fast_tests/test_scenarios.py b/tests/fast_tests/test_scenarios.py index a1d6f70b1..5fccdcb3b 100644 --- a/tests/fast_tests/test_scenarios.py +++ b/tests/fast_tests/test_scenarios.py @@ -5,6 +5,10 @@ from flow.networks import BottleneckNetwork, FigureEightNetwork, \ TrafficLightGridNetwork, HighwayNetwork, RingNetwork, MergeNetwork, \ MiniCityNetwork, MultiRingNetwork +from flow.networks import I210SubNetwork +from tests.setup_scripts import highway_exp_setup + +import flow.config as config __all__ = [ "MultiRingNetwork", "MiniCityNetwork" @@ -94,15 +98,105 @@ def test_additional_net_params(self): "length": 1000, "lanes": 4, "speed_limit": 30, - "num_edges": 1 + "num_edges": 1, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, } ) ) + def test_ghost_edge(self): + """Validate the functionality of the ghost edge feature.""" + # =================================================================== # + # Without a ghost edge # + # =================================================================== # + + # create the network + env, _, _ = highway_exp_setup( + net_params=NetParams(additional_params={ + "length": 1000, + "lanes": 4, + "speed_limit": 30, + "num_edges": 1, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, + }) + ) + env.reset() + + # check the network length + self.assertEqual(env.k.network.length(), 1000) + + # check the edge list + self.assertEqual(env.k.network.get_edge_list(), ["highway_0"]) + + # check the speed limits of the edges + self.assertEqual(env.k.network.speed_limit("highway_0"), 30) + + # =================================================================== # + # With a ghost edge (300m, 25m/s) # + # =================================================================== # + + # create the network + env, _, _ = highway_exp_setup( + net_params=NetParams(additional_params={ + "length": 1000, + "lanes": 4, + "speed_limit": 30, + "num_edges": 1, + "use_ghost_edge": True, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, + }) + ) + env.reset() + + # check the network length + self.assertEqual(env.k.network.length(), 1300.1) + + # check the edge list + self.assertEqual(env.k.network.get_edge_list(), + ["highway_0", "highway_end"]) + + # check the speed limits of the edges + self.assertEqual(env.k.network.speed_limit("highway_0"), 30) + self.assertEqual(env.k.network.speed_limit("highway_end"), 25) + + # =================================================================== # + # With a ghost edge (500m, 10m/s) # + # =================================================================== # + + # create the network + env, _, _ = highway_exp_setup( + net_params=NetParams(additional_params={ + "length": 1000, + "lanes": 4, + "speed_limit": 30, + "num_edges": 1, + "use_ghost_edge": True, + "ghost_speed_limit": 10, + "boundary_cell_length": 500, + }) + ) + env.reset() + + # check the network length + self.assertEqual(env.k.network.length(), 1500.1) + + # check the edge list + self.assertEqual(env.k.network.get_edge_list(), + ["highway_0", "highway_end"]) + + # check the speed limits of the edges + self.assertEqual(env.k.network.speed_limit("highway_0"), 30) + self.assertEqual(env.k.network.speed_limit("highway_end"), 10) + class TestRingNetwork(unittest.TestCase): - """Tests LoopNetwork in flow/networks/ring.py.""" + """Tests RingNetwork in flow/networks/ring.py.""" def test_additional_net_params(self): """Ensures that not returning the correct params leads to an error.""" @@ -142,7 +236,7 @@ def test_additional_net_params(self): class TestMultiRingNetwork(unittest.TestCase): - """Tests MultiLoopNetwork in flow/networks/multi_ring.py.""" + """Tests MultiRingNetwork in flow/networks/multi_ring.py.""" def test_additional_net_params(self): """Ensures that not returning the correct params leads to an error.""" @@ -160,6 +254,150 @@ def test_additional_net_params(self): ) +class TestI210SubNetwork(unittest.TestCase): + + """Tests I210SubNetwork in flow/networks/i210_subnetwork.py.""" + + def test_additional_net_params(self): + """Ensures that not returning the correct params leads to an error.""" + self.assertTrue( + test_additional_params( + network_class=I210SubNetwork, + additional_params={ + "on_ramp": False, + "ghost_edge": False, + } + ) + ) + + def test_specify_routes(self): + """Validates that the routes are properly specified for the network. + + This is done simply by checking the initial edges routes are specified + from, which alternates based on choice of network configuration. + + This method tests the routes for the following cases: + + 1. on_ramp = False, ghost_edge = False + 2. on_ramp = True, ghost_edge = False + 3. on_ramp = False, ghost_edge = True + 4. on_ramp = True, ghost_edge = True + """ + # test case 1 + network = I210SubNetwork( + name='test-3', + vehicles=VehicleParams(), + net_params=NetParams( + template=os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml" + ), + additional_params={ + "on_ramp": False, + "ghost_edge": False, + }, + ), + ) + + self.assertEqual( + ['119257914'], + sorted(list(network.specify_routes(network.net_params).keys())) + ) + + del network + + # test case 2 + network = I210SubNetwork( + name='test-3', + vehicles=VehicleParams(), + net_params=NetParams( + template=os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml" + ), + additional_params={ + "on_ramp": True, + "ghost_edge": True, + }, + ), + ) + + self.assertEqual( + ['119257908#0', + '119257908#1', + '119257908#1-AddedOffRampEdge', + '119257908#1-AddedOnRampEdge', + '119257908#2', + '119257908#3', + '119257914', + '173381935', + '27414342#0', + '27414342#1-AddedOnRampEdge', + '27414345', + 'ghost0'], + sorted(list(network.specify_routes(network.net_params).keys())) + ) + + del network + + # test case 3 + network = I210SubNetwork( + name='test-3', + vehicles=VehicleParams(), + net_params=NetParams( + template=os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml" + ), + additional_params={ + "on_ramp": False, + "ghost_edge": True, + }, + ), + ) + + self.assertEqual( + ['119257914', 'ghost0'], + sorted(list(network.specify_routes(network.net_params).keys())) + ) + + del network + + # test case 4 + network = I210SubNetwork( + name='test-3', + vehicles=VehicleParams(), + net_params=NetParams( + template=os.path.join( + config.PROJECT_PATH, + "examples/exp_configs/templates/sumo/test2.net.xml" + ), + additional_params={ + "on_ramp": True, + "ghost_edge": True, + }, + ), + ) + + self.assertEqual( + ['119257908#0', + '119257908#1', + '119257908#1-AddedOffRampEdge', + '119257908#1-AddedOnRampEdge', + '119257908#2', + '119257908#3', + '119257914', + '173381935', + '27414342#0', + '27414342#1-AddedOnRampEdge', + '27414345', + 'ghost0'], + sorted(list(network.specify_routes(network.net_params).keys())) + ) + + del network + + ############################################################################### # Utility methods # ############################################################################### diff --git a/tests/fast_tests/test_green_wave.py b/tests/fast_tests/test_traffic_light_grid.py similarity index 94% rename from tests/fast_tests/test_green_wave.py rename to tests/fast_tests/test_traffic_light_grid.py index fd14a20f6..224f3e53f 100644 --- a/tests/fast_tests/test_green_wave.py +++ b/tests/fast_tests/test_traffic_light_grid.py @@ -8,11 +8,12 @@ class Test1x1Environment(unittest.TestCase): def setUp(self): # create the environment and network classes for a traffic light grid network - self.env, _ = traffic_light_grid_mxn_exp_setup() + + self.env, _, flow_params = traffic_light_grid_mxn_exp_setup() self.env.reset() # instantiate an experiment class - self.exp = Experiment(self.env) + self.exp = Experiment(flow_params) def tearDown(self): # terminate the traci instance @@ -94,11 +95,12 @@ def test_get_distance_to_intersection(self): class Test2x2Environment(unittest.TestCase): def setUp(self): # create the environment and network classes for a traffic light grid network - self.env, _ = traffic_light_grid_mxn_exp_setup(row_num=2, col_num=2) + + self.env, _, flow_params = traffic_light_grid_mxn_exp_setup(row_num=2, col_num=2) self.env.reset() # instantiate an experiment class - self.exp = Experiment(self.env) + self.exp = Experiment(flow_params) def tearDown(self): # terminate the traci instance diff --git a/tests/fast_tests/test_traffic_lights.py b/tests/fast_tests/test_traffic_lights.py index a06d34a0d..d5c02b5bc 100644 --- a/tests/fast_tests/test_traffic_lights.py +++ b/tests/fast_tests/test_traffic_lights.py @@ -40,7 +40,7 @@ def test_single_lane(self): net_params = NetParams(additional_params=additional_net_params) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup( + self.env, _, _ = ring_road_exp_setup( net_params=net_params, traffic_lights=traffic_lights) self.env.reset() @@ -65,7 +65,7 @@ def test_multi_lane(self): net_params = NetParams(additional_params=additional_net_params) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup( + self.env, _, _ = ring_road_exp_setup( net_params=net_params, traffic_lights=traffic_lights) self.env.reset() @@ -96,7 +96,7 @@ def setUp(self): net_params = NetParams(additional_params=additional_net_params) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup( + self.env, _, _ = ring_road_exp_setup( net_params=net_params, traffic_lights=traffic_lights) def tearDown(self): @@ -153,7 +153,7 @@ def setUp(self): min_gap=2.5, tau=1.1), num_vehicles=16) - self.env, _ = traffic_light_grid_mxn_exp_setup( + self.env, _, _ = traffic_light_grid_mxn_exp_setup( row_num=1, col_num=3, vehicles=vehicles) def tearDown(self): @@ -228,17 +228,18 @@ def setUp(self): min_gap=2.5, tau=1.1), num_vehicles=16) - env, _ = traffic_light_grid_mxn_exp_setup( + _, _, flow_params = traffic_light_grid_mxn_exp_setup( row_num=1, col_num=3, vehicles=vehicles) - self.exp = Experiment(env) + flow_params['env'].horizon = 50 + self.exp = Experiment(flow_params) def tearDown(self): # free data used by the class self.exp = None def test_it_runs(self): - self.exp.run(5, 50) + self.exp.run(5) class TestIndividualLights(unittest.TestCase): @@ -285,17 +286,18 @@ def setUp(self): file="testindividuallights.xml", freq=100) - env, _ = traffic_light_grid_mxn_exp_setup( + _, _, flow_params = traffic_light_grid_mxn_exp_setup( row_num=1, col_num=4, tl_logic=tl_logic) - self.exp = Experiment(env) + flow_params['env'].horizon = 50 + self.exp = Experiment(flow_params) def tearDown(self): # free data used by the class self.exp = None def test_it_runs(self): - self.exp.run(5, 50) + self.exp.run(5) class TestCustomization(unittest.TestCase): @@ -330,7 +332,7 @@ def setUp(self): net_params = NetParams(additional_params=additional_net_params) # create the environment and network classes for a ring road - self.env, _ = ring_road_exp_setup( + self.env, _, _ = ring_road_exp_setup( net_params=net_params, traffic_lights=traffic_lights) def tearDown(self): diff --git a/tests/fast_tests/test_util.py b/tests/fast_tests/test_util.py index 458fadcf4..67386cc77 100644 --- a/tests/fast_tests/test_util.py +++ b/tests/fast_tests/test_util.py @@ -14,7 +14,6 @@ from flow.core.util import emission_to_csv from flow.envs import MergePOEnv from flow.networks import MergeNetwork -from flow.utils.flow_warnings import deprecated_attribute from flow.utils.registry import make_create_env from flow.utils.rllib import FlowParamsEncoder, get_flow_params @@ -60,25 +59,6 @@ def test_emission_to_csv(self): self.assertEqual(len(dict1), 104) -class TestWarnings(unittest.TestCase): - """Tests warning functions located in flow.utils.warnings""" - - def test_deprecated_attribute(self): - # dummy class - class Foo(object): - pass - - # dummy attribute name - dep_from = "bar_deprecated" - dep_to = "bar_new" - - # check the deprecation warning is printing what is expected - self.assertWarnsRegex( - PendingDeprecationWarning, - "The attribute bar_deprecated in Foo is deprecated, use bar_new " - "instead.", deprecated_attribute, Foo(), dep_from, dep_to) - - class TestRegistry(unittest.TestCase): """Tests the methods located in flow/utils/registry.py""" diff --git a/tests/fast_tests/test_vehicles.py b/tests/fast_tests/test_vehicles.py index 593c8ccef..7e1405007 100644 --- a/tests/fast_tests/test_vehicles.py +++ b/tests/fast_tests/test_vehicles.py @@ -33,7 +33,7 @@ def test_speed_lane_change_modes(self): speed_mode='obey_safe_speed', ), lane_change_params=SumoLaneChangeParams( - lane_change_mode="no_lat_collide", + lane_change_mode="no_lc_safe", ) ) @@ -56,7 +56,7 @@ def test_speed_lane_change_modes(self): self.assertEqual(vehicles.type_parameters["typeB"][ "car_following_params"].speed_mode, 0) self.assertEqual(vehicles.type_parameters["typeB"][ - "lane_change_params"].lane_change_mode, 1621) + "lane_change_params"].lane_change_mode, 512) vehicles.add( "typeC", @@ -89,7 +89,7 @@ def test_controlled_id_params(self): speed_mode="obey_safe_speed", ), lane_change_params=SumoLaneChangeParams( - lane_change_mode="no_lat_collide", + lane_change_mode="no_lc_safe", )) default_mingap = SumoCarFollowingParams().controller_params["minGap"] self.assertEqual(vehicles.types[0]["type_params"]["minGap"], @@ -118,7 +118,7 @@ def test_add_vehicles_human(self): num_vehicles=4, acceleration_controller=(IDMController, {})) - env, _ = ring_road_exp_setup(vehicles=vehicles) + env, _, _ = ring_road_exp_setup(vehicles=vehicles) self.assertEqual(env.k.vehicle.num_vehicles, 7) self.assertEqual(len(env.k.vehicle.get_ids()), 7) @@ -138,7 +138,7 @@ def test_add_vehicles_rl(self): num_vehicles=10, acceleration_controller=(RLController, {})) - env, _ = ring_road_exp_setup(vehicles=vehicles) + env, _, _ = ring_road_exp_setup(vehicles=vehicles) self.assertEqual(env.k.vehicle.num_vehicles, 10) self.assertEqual(len(env.k.vehicle.get_ids()), 10) @@ -160,7 +160,7 @@ def test_remove(self): num_vehicles=10, acceleration_controller=(RLController, {})) - env, _ = ring_road_exp_setup(vehicles=vehicles) + env, _, _ = ring_road_exp_setup(vehicles=vehicles) # remove one human-driven vehicle and on rl vehicle env.k.vehicle.remove("test_0") @@ -225,7 +225,7 @@ def test_no_junctions_ring(self): initial_config = InitialConfig(lanes_distribution=float("inf")) - env, _ = ring_road_exp_setup( + env, _, _ = ring_road_exp_setup( net_params=net_params, vehicles=vehicles, initial_config=initial_config) @@ -258,7 +258,10 @@ def test_no_junctions_highway(self): "lanes": 3, "speed_limit": 30, "resolution": 40, - "num_edges": 1 + "num_edges": 1, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, } net_params = NetParams(additional_params=additional_net_params) vehicles = VehicleParams() @@ -279,7 +282,7 @@ def test_no_junctions_highway(self): "start_lanes": [1, 2, 0]} initial_config.additional_params = initial_pos - env, _ = highway_exp_setup( + env, _, _ = highway_exp_setup( sim_params=SumoParams(sim_step=0.1, render=False), net_params=net_params, vehicles=vehicles, @@ -330,7 +333,10 @@ def test_no_junctions_highway(self): "lanes": 4, "speed_limit": 30, "resolution": 40, - "num_edges": 1 + "num_edges": 1, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, } net_params = NetParams(additional_params=additional_net_params) vehicles = VehicleParams() @@ -355,7 +361,7 @@ def test_no_junctions_highway(self): "start_lanes": [0, 0, 0, 1, 1, 2, 2, 3, 3]} initial_config.additional_params = initial_pos - env, _ = highway_exp_setup( + env, _, _ = highway_exp_setup( sim_params=SumoParams(sim_step=0.1, render=False), net_params=net_params, vehicles=vehicles, @@ -398,7 +404,10 @@ def test_no_junctions_highway(self): "lanes": 3, "speed_limit": 30, "resolution": 40, - "num_edges": 3 + "num_edges": 3, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, } net_params = NetParams(additional_params=additional_net_params) vehicles = VehicleParams() @@ -419,7 +428,7 @@ def test_no_junctions_highway(self): "start_lanes": [1, 2, 0]} initial_config.additional_params = initial_pos - env, _ = highway_exp_setup( + env, _, _ = highway_exp_setup( sim_params=SumoParams(sim_step=0.1, render=False), net_params=net_params, vehicles=vehicles, @@ -465,7 +474,10 @@ def test_no_junctions_highway(self): "lanes": 3, "speed_limit": 30, "resolution": 40, - "num_edges": 3 + "num_edges": 3, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, } net_params = NetParams(additional_params=additional_net_params) vehicles = VehicleParams() @@ -486,7 +498,7 @@ def test_no_junctions_highway(self): "start_lanes": [0, 0, 0]} initial_config.additional_params = initial_pos - env, _ = highway_exp_setup( + env, _, _ = highway_exp_setup( sim_params=SumoParams(sim_step=0.1, render=False), net_params=net_params, vehicles=vehicles, @@ -542,7 +554,7 @@ def setUp(self): vehicles = VehicleParams() vehicles.add(veh_id="test", num_vehicles=20) - self.env, _ = ring_road_exp_setup(vehicles=vehicles) + self.env, _, _ = ring_road_exp_setup(vehicles=vehicles) def tearDown(self): # free data used by the class @@ -563,7 +575,7 @@ def test_obs_ids(self): vehicles = VehicleParams() vehicles.add(veh_id="test", num_vehicles=10) - env, _ = ring_road_exp_setup(vehicles=vehicles) + env, _, _ = ring_road_exp_setup(vehicles=vehicles) # test setting new observed values env.k.vehicle.set_observed("test_0") diff --git a/tests/fast_tests/test_visualizers.py b/tests/fast_tests/test_visualizers.py index 3117f37f7..d2f4a20a4 100644 --- a/tests/fast_tests/test_visualizers.py +++ b/tests/fast_tests/test_visualizers.py @@ -1,5 +1,3 @@ -# from flow.visualize import visualizer_rllab as vs_rllab -# from flow.visualize.visualizer_rllab import visualizer_rllab from flow.visualize import visualizer_rllib as vs_rllib from flow.visualize.visualizer_rllib import visualizer_rllib import flow.visualize.capacity_diagram_generator as cdg @@ -55,19 +53,6 @@ def test_visualizer_multi(self): visualizer_rllib(pass_args) -# class TestVisualizerRLlab(unittest.TestCase): -# """Tests visualizer_rllab""" -# -# def test_visualizer(self): -# # current path -# current_path = os.path.realpath(__file__).rsplit('/', 1)[0] -# arg_str = '{}/../data/rllab_data/itr_0.pkl --num_rollouts 1 ' \ -# '--no_render'.format(current_path).split() -# parser = vs_rllab.create_parser() -# pass_args = parser.parse_args(arg_str) -# visualizer_rllab(pass_args) - - class TestPlotters(unittest.TestCase): def test_capacity_diagram_generator(self): @@ -106,193 +91,226 @@ def test_capacity_diagram_generator(self): np.testing.assert_array_almost_equal(std_outflows, expected_stds) def test_time_space_diagram_figure_eight(self): - # check that the exported data matches the expected emission file data - fig8_emission_data = { - 'idm_3': {'pos': [27.25, 28.25, 30.22, 33.17], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['upper_ring', 'upper_ring', 'upper_ring', - 'upper_ring']}, - 'idm_4': {'pos': [56.02, 57.01, 58.99, 61.93], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['upper_ring', 'upper_ring', 'upper_ring', - 'upper_ring']}, - 'idm_5': {'pos': [84.79, 85.78, 87.76, 90.7], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['upper_ring', 'upper_ring', 'upper_ring', - 'upper_ring']}, - 'idm_2': {'pos': [28.77, 29.76, 1.63, 4.58], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.97, 2.95], - 'edge': ['top', 'top', 'upper_ring', 'upper_ring']}, - 'idm_13': {'pos': [106.79, 107.79, 109.77, 112.74], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.96], - 'edge': ['lower_ring', 'lower_ring', 'lower_ring', - 'lower_ring']}, - 'idm_9': {'pos': [22.01, 23.0, 24.97, 27.92], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.97, 2.95], - 'edge': ['left', 'left', 'left', 'left']}, - 'idm_6': {'pos': [113.56, 114.55, 116.52, 119.47], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.97, 2.95], - 'edge': ['upper_ring', 'upper_ring', 'upper_ring', - 'upper_ring']}, - 'idm_8': {'pos': [29.44, 0.28, 2.03, 4.78], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.84, 1.76, 2.75], - 'edge': ['right', ':center_0', ':center_0', - ':center_0']}, - 'idm_12': {'pos': [78.03, 79.02, 80.99, 83.94], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['lower_ring', 'lower_ring', 'lower_ring', - 'lower_ring']}, - 'idm_10': {'pos': [20.49, 21.48, 23.46, 26.41], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['lower_ring', 'lower_ring', 'lower_ring', - 'lower_ring']}, - 'idm_11': {'pos': [49.26, 50.25, 52.23, 55.17], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['lower_ring', 'lower_ring', 'lower_ring', - 'lower_ring']}, - 'idm_1': {'pos': [0.0, 0.99, 2.97, 5.91], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.98, 2.95], - 'edge': ['top', 'top', 'top', 'top']}, - 'idm_7': {'pos': [0.67, 1.66, 3.64, 6.58], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 0.99, 1.97, 2.94], - 'edge': ['right', 'right', 'right', 'right']}, - 'idm_0': {'pos': [0.0, 1.0, 2.98, 5.95], - 'time': [1.0, 2.0, 3.0, 4.0], - 'vel': [0.0, 1.0, 1.99, 2.97], - 'edge': ['bottom', 'bottom', 'bottom', 'bottom']} - } dir_path = os.path.dirname(os.path.realpath(__file__)) - actual_emission_data = tsd.import_data_from_emission( - os.path.join(dir_path, 'test_files/fig8_emission.csv')) - self.assertDictEqual(fig8_emission_data, actual_emission_data) - - # test get_time_space_data for figure eight networks flow_params = tsd.get_flow_params( os.path.join(dir_path, 'test_files/fig8.json')) - pos, speed, _ = tsd.get_time_space_data( - actual_emission_data, flow_params) - - expected_pos = np.array( - [[60, 23.8, 182.84166941, 154.07166941, 125.30166941, 96.54166941, - -203.16166941, -174.40166941, -145.63166941, -116.86166941, - -88.09166941, -59.33, -30.56, -1.79], - [59, 22.81, 181.85166941, 153.08166941, 124.31166941, 95.54166941, - -202.17166941, -173.40166941, -144.64166941, -115.87166941, - -87.10166941, -58.34, -29.72, -0.8], - [57.02, 20.83, 179.87166941, 151.10166941, 122.34166941, - 93.56166941, -200.02166941, -171.43166941, -142.66166941, - -113.89166941, -85.13166941, -56.36, -27.97, 208.64166941]] + emission_data = tsd.import_data_from_trajectory( + os.path.join(dir_path, 'test_files/fig8_emission.csv'), flow_params) + + segs, _ = tsd.get_time_space_data(emission_data, flow_params) + + expected_segs = np.array([ + [[1., 60.], [2., 59.]], + [[2., 59.], [3., 57.02]], + [[3., 57.02], [4., 54.05]], + [[1., 23.8], [2., 22.81]], + [[2., 22.81], [3., 20.83]], + [[3., 20.83], [4., 17.89]], + [[1., 182.84166941], [2., 181.85166941]], + [[2., 181.85166941], [3., 179.87166941]], + [[3., 179.87166941], [4., 176.92166941]], + [[1., 154.07166941], [2., 153.08166941]], + [[2., 153.08166941], [3., 151.10166941]], + [[3., 151.10166941], [4., 148.16166941]], + [[1., 125.30166941], [2., 124.31166941]], + [[2., 124.31166941], [3., 122.34166941]], + [[3., 122.34166941], [4., 119.39166941]], + [[1., 96.54166941], [2., 95.54166941]], + [[2., 95.54166941], [3., 93.56166941]], + [[3., 93.56166941], [4., 90.59166941]], + [[1., -203.16166941], [2., -202.17166941]], + [[2., -202.17166941], [3., -200.02166941]], + [[3., -200.02166941], [4., -197.07166941]], + [[1., -174.40166941], [2., -173.40166941]], + [[2., -173.40166941], [3., -171.43166941]], + [[3., -171.43166941], [4., -168.48166941]], + [[1., -145.63166941], [2., -144.64166941]], + [[2., -144.64166941], [3., -142.66166941]], + [[3., -142.66166941], [4., -139.72166941]], + [[1., -116.86166941], [2., -115.87166941]], + [[2., -115.87166941], [3., -113.89166941]], + [[3., -113.89166941], [4., -110.95166941]], + [[1., -88.09166941], [2., -87.10166941]], + [[2., -87.10166941], [3., -85.13166941]], + [[3., -85.13166941], [4., -82.18166941]], + [[1., -59.33], [2., -58.34]], + [[2., -58.34], [3., -56.36]], + [[3., -56.36], [4., -53.42]], + [[1., -30.56], [2., -29.72]], + [[2., -29.72], [3., -27.97]], + [[3., -27.97], [4., -25.22]], + [[1., -1.79], [2., -0.8]], + [[2., -0.8], [3., 208.64166941]], + [[3., 208.64166941], [4., 205.69166941]]] ) - expected_speed = np.array([ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [1, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, 0.99, - 0.99, 0.84, 0.99], - [1.99, 1.98, 1.98, 1.98, 1.98, 1.98, 1.97, 1.98, 1.98, 1.98, 1.97, - 1.97, 1.76, 1.97] - ]) - np.testing.assert_array_almost_equal(pos[:-1, :], expected_pos) - np.testing.assert_array_almost_equal(speed[:-1, :], expected_speed) + np.testing.assert_array_almost_equal(segs, expected_segs) def test_time_space_diagram_merge(self): dir_path = os.path.dirname(os.path.realpath(__file__)) - emission_data = tsd.import_data_from_emission( - os.path.join(dir_path, 'test_files/merge_emission.csv')) - flow_params = tsd.get_flow_params( os.path.join(dir_path, 'test_files/merge.json')) - pos, speed, _ = tsd.get_time_space_data(emission_data, flow_params) - - expected_pos = np.array( - [[4.86, 180.32, 361.32, 547.77, 0], - [4.88, 180.36, 361.36, 547.8, 0], - [4.95, 180.43, 361.44, 547.87, 0], - [5.06, 180.54, 361.56, 547.98, 0], - [5.21, 180.68, 361.72, 548.12, 0], - [5.4, 180.86, 0, 0, 0]] - ) - expected_speed = np.array( - [[0, 0, 0, 0, 0], - [0.15, 0.17, 0.19, 0.14, 0], - [0.35, 0.37, 0.39, 0.34, 0], - [0.54, 0.57, 0.59, 0.54, 0], - [0.74, 0.7, 0.79, 0.71, 0], - [0.94, 0.9, 0, 0, 0]] + emission_data = tsd.import_data_from_trajectory( + os.path.join(dir_path, 'test_files/merge_emission.csv'), flow_params) + + segs, _ = tsd.get_time_space_data(emission_data, flow_params) + + expected_segs = np.array([ + [[2.0000e-01, 7.2949e+02], [4.0000e-01, 7.2953e+02]], + [[4.0000e-01, 7.2953e+02], [6.0000e-01, 7.2961e+02]], + [[6.0000e-01, 7.2961e+02], [8.0000e-01, 7.2973e+02]], + [[8.0000e-01, 7.2973e+02], [1.0000e+00, 7.2988e+02]]] ) - np.testing.assert_array_almost_equal(pos, expected_pos) - np.testing.assert_array_almost_equal(speed, expected_speed) + np.testing.assert_array_almost_equal(segs, expected_segs) - def test_time_space_diagram_ring_road(self): + def test_time_space_diagram_I210(self): dir_path = os.path.dirname(os.path.realpath(__file__)) - emission_data = tsd.import_data_from_emission( - os.path.join(dir_path, 'test_files/ring_230_emission.csv')) + module = __import__("examples.exp_configs.non_rl", fromlist=["i210_subnetwork"]) + flow_params = getattr(module, "i210_subnetwork").flow_params + emission_data = tsd.import_data_from_trajectory( + os.path.join(dir_path, 'test_files/i210_emission.csv'), flow_params) + + segs, _ = tsd.get_time_space_data(emission_data, flow_params) + + expected_segs = { + 1: np.array([ + [[0.8, 5.1], [1.6, 23.37]], + [[1.6, 23.37], [2.4, 42.02]], + [[2.4, 42.02], [3.2, 61.21]], + [[3.2, 61.21], [4., 18.87]], + [[4., 18.87], [4.8, 39.93]], + [[2.4, 5.1], [3.2, 22.97]], + [[3.2, 22.97], [4., 40.73]]] + ), + 2: np.array([ + [[2.4, 5.1], [3.2, 23.98]], + [[3.2, 23.98], [4., 43.18]]] + ), + 3: np.array([ + [[0.8, 5.1], [1.6, 23.72]], + [[1.6, 23.72], [2.4, 43.06]], + [[2.4, 43.06], [3.2, 1.33]], + [[3.2, 1.33], [4., 21.65]], + [[4., 21.65], [4.8, 43.46]], + [[2.4, 5.1], [3.2, 23.74]], + [[3.2, 23.74], [4., 42.38]]] + ), + 4: np.array([ + [[2.4, 5.1], [3.2, 23.6]], + [[3.2, 23.6], [4., 42.46]]] + )} + + for lane, expected_seg in expected_segs.items(): + np.testing.assert_array_almost_equal(segs[lane], expected_seg) + def test_time_space_diagram_ring_road(self): + dir_path = os.path.dirname(os.path.realpath(__file__)) flow_params = tsd.get_flow_params( os.path.join(dir_path, 'test_files/ring_230.json')) - pos, speed, _ = tsd.get_time_space_data(emission_data, flow_params) - - expected_pos = np.array( - [[0.0000e+00, 9.5500e+00, 9.5550e+01, 1.0510e+02, 1.1465e+02, - 1.2429e+02, 1.3384e+02, 1.4338e+02, 1.5293e+02, 1.6247e+02, - 1.7202e+02, 1.8166e+02, 1.9090e+01, 1.9121e+02, 2.0075e+02, - 2.8640e+01, 3.8180e+01, 4.7730e+01, 5.7270e+01, 6.6920e+01, - 7.6460e+01, 8.6010e+01], - [1.0000e-02, 9.5500e+00, 9.5560e+01, 1.0511e+02, 1.1465e+02, - 1.2430e+02, 1.3384e+02, 1.4339e+02, 1.5294e+02, 1.6248e+02, - 1.7203e+02, 1.8167e+02, 1.9100e+01, 1.9122e+02, 2.0076e+02, - 2.8640e+01, 3.8190e+01, 4.7740e+01, 5.7280e+01, 6.6930e+01, - 7.6470e+01, 8.6020e+01], - [2.0000e-02, 9.5700e+00, 9.5580e+01, 1.0512e+02, 1.1467e+02, - 1.2431e+02, 1.3386e+02, 1.4341e+02, 1.5295e+02, 1.6250e+02, - 1.7204e+02, 1.8169e+02, 1.9110e+01, 1.9123e+02, 2.0078e+02, - 2.8660e+01, 3.8210e+01, 4.7750e+01, 5.7300e+01, 6.6940e+01, - 7.6490e+01, 8.6030e+01], - [5.0000e-02, 9.5900e+00, 9.5600e+01, 1.0515e+02, 1.1469e+02, - 1.2434e+02, 1.3388e+02, 1.4343e+02, 1.5297e+02, 1.6252e+02, - 1.7207e+02, 1.8171e+02, 1.9140e+01, 1.9126e+02, 2.0081e+02, - 2.8680e+01, 3.8230e+01, 4.7770e+01, 5.7320e+01, 6.6970e+01, - 7.6510e+01, 8.6060e+01], - [8.0000e-02, 9.6200e+00, 9.5630e+01, 1.0518e+02, 1.1472e+02, - 1.2437e+02, 1.3391e+02, 1.4346e+02, 1.5301e+02, 1.6255e+02, - 1.7210e+02, 1.8174e+02, 1.9170e+01, 1.9129e+02, 2.0085e+02, - 2.8710e+01, 3.8260e+01, 4.7810e+01, 5.7350e+01, 6.7000e+01, - 7.6540e+01, 8.6090e+01], - [1.2000e-01, 9.6600e+00, 9.5670e+01, 1.0522e+02, 1.1476e+02, - 1.2441e+02, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, - 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, - 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, 0.0000e+00, - 0.0000e+00, 0.0000e+00]] + emission_data = tsd.import_data_from_trajectory( + os.path.join(dir_path, 'test_files/ring_230_emission.csv'), flow_params) + + segs, _ = tsd.get_time_space_data(emission_data, flow_params) + + expected_segs = np.array([ + [[1.0000e-01, 0.0000e+00], [2.0000e-01, 1.0000e-02]], + [[2.0000e-01, 1.0000e-02], [3.0000e-01, 2.0000e-02]], + [[3.0000e-01, 2.0000e-02], [4.0000e-01, 5.0000e-02]], + [[4.0000e-01, 5.0000e-02], [5.0000e-01, 8.0000e-02]], + [[5.0000e-01, 8.0000e-02], [6.0000e-01, 1.2000e-01]], + [[1.0000e-01, 9.5500e+00], [2.0000e-01, 9.5500e+00]], + [[2.0000e-01, 9.5500e+00], [3.0000e-01, 9.5700e+00]], + [[3.0000e-01, 9.5700e+00], [4.0000e-01, 9.5900e+00]], + [[4.0000e-01, 9.5900e+00], [5.0000e-01, 9.6200e+00]], + [[5.0000e-01, 9.6200e+00], [6.0000e-01, 9.6600e+00]], + [[1.0000e-01, 9.5550e+01], [2.0000e-01, 9.5560e+01]], + [[2.0000e-01, 9.5560e+01], [3.0000e-01, 9.5580e+01]], + [[3.0000e-01, 9.5580e+01], [4.0000e-01, 9.5600e+01]], + [[4.0000e-01, 9.5600e+01], [5.0000e-01, 9.5630e+01]], + [[5.0000e-01, 9.5630e+01], [6.0000e-01, 9.5670e+01]], + [[1.0000e-01, 1.0510e+02], [2.0000e-01, 1.0511e+02]], + [[2.0000e-01, 1.0511e+02], [3.0000e-01, 1.0512e+02]], + [[3.0000e-01, 1.0512e+02], [4.0000e-01, 1.0515e+02]], + [[4.0000e-01, 1.0515e+02], [5.0000e-01, 1.0518e+02]], + [[5.0000e-01, 1.0518e+02], [6.0000e-01, 1.0522e+02]], + [[1.0000e-01, 1.1465e+02], [2.0000e-01, 1.1465e+02]], + [[2.0000e-01, 1.1465e+02], [3.0000e-01, 1.1467e+02]], + [[3.0000e-01, 1.1467e+02], [4.0000e-01, 1.1469e+02]], + [[4.0000e-01, 1.1469e+02], [5.0000e-01, 1.1472e+02]], + [[5.0000e-01, 1.1472e+02], [6.0000e-01, 1.1476e+02]], + [[1.0000e-01, 1.2429e+02], [2.0000e-01, 1.2430e+02]], + [[2.0000e-01, 1.2430e+02], [3.0000e-01, 1.2431e+02]], + [[3.0000e-01, 1.2431e+02], [4.0000e-01, 1.2434e+02]], + [[4.0000e-01, 1.2434e+02], [5.0000e-01, 1.2437e+02]], + [[5.0000e-01, 1.2437e+02], [6.0000e-01, 1.2441e+02]], + [[1.0000e-01, 1.3384e+02], [2.0000e-01, 1.3384e+02]], + [[2.0000e-01, 1.3384e+02], [3.0000e-01, 1.3386e+02]], + [[3.0000e-01, 1.3386e+02], [4.0000e-01, 1.3388e+02]], + [[4.0000e-01, 1.3388e+02], [5.0000e-01, 1.3391e+02]], + [[1.0000e-01, 1.4338e+02], [2.0000e-01, 1.4339e+02]], + [[2.0000e-01, 1.4339e+02], [3.0000e-01, 1.4341e+02]], + [[3.0000e-01, 1.4341e+02], [4.0000e-01, 1.4343e+02]], + [[4.0000e-01, 1.4343e+02], [5.0000e-01, 1.4346e+02]], + [[1.0000e-01, 1.5293e+02], [2.0000e-01, 1.5294e+02]], + [[2.0000e-01, 1.5294e+02], [3.0000e-01, 1.5295e+02]], + [[3.0000e-01, 1.5295e+02], [4.0000e-01, 1.5297e+02]], + [[4.0000e-01, 1.5297e+02], [5.0000e-01, 1.5301e+02]], + [[1.0000e-01, 1.6247e+02], [2.0000e-01, 1.6248e+02]], + [[2.0000e-01, 1.6248e+02], [3.0000e-01, 1.6250e+02]], + [[3.0000e-01, 1.6250e+02], [4.0000e-01, 1.6252e+02]], + [[4.0000e-01, 1.6252e+02], [5.0000e-01, 1.6255e+02]], + [[1.0000e-01, 1.7202e+02], [2.0000e-01, 1.7203e+02]], + [[2.0000e-01, 1.7203e+02], [3.0000e-01, 1.7204e+02]], + [[3.0000e-01, 1.7204e+02], [4.0000e-01, 1.7207e+02]], + [[4.0000e-01, 1.7207e+02], [5.0000e-01, 1.7210e+02]], + [[1.0000e-01, 1.8166e+02], [2.0000e-01, 1.8167e+02]], + [[2.0000e-01, 1.8167e+02], [3.0000e-01, 1.8169e+02]], + [[3.0000e-01, 1.8169e+02], [4.0000e-01, 1.8171e+02]], + [[4.0000e-01, 1.8171e+02], [5.0000e-01, 1.8174e+02]], + [[1.0000e-01, 1.9090e+01], [2.0000e-01, 1.9100e+01]], + [[2.0000e-01, 1.9100e+01], [3.0000e-01, 1.9110e+01]], + [[3.0000e-01, 1.9110e+01], [4.0000e-01, 1.9140e+01]], + [[4.0000e-01, 1.9140e+01], [5.0000e-01, 1.9170e+01]], + [[1.0000e-01, 1.9121e+02], [2.0000e-01, 1.9122e+02]], + [[2.0000e-01, 1.9122e+02], [3.0000e-01, 1.9123e+02]], + [[3.0000e-01, 1.9123e+02], [4.0000e-01, 1.9126e+02]], + [[4.0000e-01, 1.9126e+02], [5.0000e-01, 1.9129e+02]], + [[1.0000e-01, 2.0075e+02], [2.0000e-01, 2.0076e+02]], + [[2.0000e-01, 2.0076e+02], [3.0000e-01, 2.0078e+02]], + [[3.0000e-01, 2.0078e+02], [4.0000e-01, 2.0081e+02]], + [[4.0000e-01, 2.0081e+02], [5.0000e-01, 2.0085e+02]], + [[1.0000e-01, 2.8640e+01], [2.0000e-01, 2.8640e+01]], + [[2.0000e-01, 2.8640e+01], [3.0000e-01, 2.8660e+01]], + [[3.0000e-01, 2.8660e+01], [4.0000e-01, 2.8680e+01]], + [[4.0000e-01, 2.8680e+01], [5.0000e-01, 2.8710e+01]], + [[1.0000e-01, 3.8180e+01], [2.0000e-01, 3.8190e+01]], + [[2.0000e-01, 3.8190e+01], [3.0000e-01, 3.8210e+01]], + [[3.0000e-01, 3.8210e+01], [4.0000e-01, 3.8230e+01]], + [[4.0000e-01, 3.8230e+01], [5.0000e-01, 3.8260e+01]], + [[1.0000e-01, 4.7730e+01], [2.0000e-01, 4.7740e+01]], + [[2.0000e-01, 4.7740e+01], [3.0000e-01, 4.7750e+01]], + [[3.0000e-01, 4.7750e+01], [4.0000e-01, 4.7770e+01]], + [[4.0000e-01, 4.7770e+01], [5.0000e-01, 4.7810e+01]], + [[1.0000e-01, 5.7270e+01], [2.0000e-01, 5.7280e+01]], + [[2.0000e-01, 5.7280e+01], [3.0000e-01, 5.7300e+01]], + [[3.0000e-01, 5.7300e+01], [4.0000e-01, 5.7320e+01]], + [[4.0000e-01, 5.7320e+01], [5.0000e-01, 5.7350e+01]], + [[1.0000e-01, 6.6920e+01], [2.0000e-01, 6.6930e+01]], + [[2.0000e-01, 6.6930e+01], [3.0000e-01, 6.6940e+01]], + [[3.0000e-01, 6.6940e+01], [4.0000e-01, 6.6970e+01]], + [[4.0000e-01, 6.6970e+01], [5.0000e-01, 6.7000e+01]], + [[1.0000e-01, 7.6460e+01], [2.0000e-01, 7.6470e+01]], + [[2.0000e-01, 7.6470e+01], [3.0000e-01, 7.6490e+01]], + [[3.0000e-01, 7.6490e+01], [4.0000e-01, 7.6510e+01]], + [[4.0000e-01, 7.6510e+01], [5.0000e-01, 7.6540e+01]], + [[1.0000e-01, 8.6010e+01], [2.0000e-01, 8.6020e+01]], + [[2.0000e-01, 8.6020e+01], [3.0000e-01, 8.6030e+01]], + [[3.0000e-01, 8.6030e+01], [4.0000e-01, 8.6060e+01]], + [[4.0000e-01, 8.6060e+01], [5.0000e-01, 8.6090e+01]]] ) - expected_speed = np.array([ - [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - [0.08, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08, - 0.08, 0.08, 0.08, 0.1, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08, 0.08], - [0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, - 0.16, 0.16, 0.16, 0.2, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16, 0.16], - [0.23, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23, - 0.23, 0.23, 0.23, 0.29, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23, 0.23], - [0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, - 0.31, 0.31, 0.31, 0.39, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31, 0.31], - [0.41, 0.41, 0.41, 0.41, 0.41, 0.41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0] - ]) - - np.testing.assert_array_almost_equal(pos, expected_pos) - np.testing.assert_array_almost_equal(speed, expected_speed) + + np.testing.assert_array_almost_equal(segs, expected_segs) def test_plot_ray_results(self): dir_path = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/setup_scripts.py b/tests/setup_scripts.py index 491e0e240..343bad906 100644 --- a/tests/setup_scripts.py +++ b/tests/setup_scripts.py @@ -15,11 +15,12 @@ from flow.core.params import TrafficLightParams from flow.core.params import VehicleParams from flow.envs.traffic_light_grid import TrafficLightGridTestEnv -from flow.envs.ring.accel import AccelEnv + from flow.networks.figure_eight import FigureEightNetwork from flow.networks.traffic_light_grid import TrafficLightGridNetwork from flow.networks.highway import HighwayNetwork from flow.networks.ring import RingNetwork +from flow.envs.ring.accel import AccelEnv def ring_road_exp_setup(sim_params=None, @@ -97,6 +98,41 @@ def ring_road_exp_setup(sim_params=None, # set default to no traffic lights traffic_lights = TrafficLightParams() + flow_params = dict( + # name of the experiment + exp_tag="RingRoadTest", + + # name of the flow environment the experiment is running on + env_name=AccelEnv, + + # name of the network class the experiment is running on + network=RingNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=sim_params, + + # environment related parameters (see flow.core.params.EnvParams) + env=env_params, + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=net_params, + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=initial_config, + + # traffic lights to be introduced to specific nodes (see + # flow.core.params.TrafficLightParams) + tls=traffic_lights, + ) + # create the network network = RingNetwork( name="RingRoadTest", @@ -112,7 +148,7 @@ def ring_road_exp_setup(sim_params=None, # reset the environment env.reset() - return env, network + return env, network, flow_params def figure_eight_exp_setup(sim_params=None, @@ -190,6 +226,41 @@ def figure_eight_exp_setup(sim_params=None, # set default to no traffic lights traffic_lights = TrafficLightParams() + flow_params = dict( + # name of the experiment + exp_tag="FigureEightTest", + + # name of the flow environment the experiment is running on + env_name=AccelEnv, + + # name of the network class the experiment is running on + network=FigureEightNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=sim_params, + + # environment related parameters (see flow.core.params.EnvParams) + env=env_params, + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=net_params, + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=initial_config, + + # traffic lights to be introduced to specific nodes (see + # flow.core.params.TrafficLightParams) + tls=traffic_lights, + ) + # create the network network = FigureEightNetwork( name="FigureEightTest", @@ -205,7 +276,7 @@ def figure_eight_exp_setup(sim_params=None, # reset the environment env.reset() - return env, network + return env, network, flow_params def highway_exp_setup(sim_params=None, @@ -272,7 +343,10 @@ def highway_exp_setup(sim_params=None, "lanes": 1, "speed_limit": 30, "resolution": 40, - "num_edges": 1 + "num_edges": 1, + "use_ghost_edge": False, + "ghost_speed_limit": 25, + "boundary_cell_length": 300, } net_params = NetParams(additional_params=additional_net_params) @@ -284,6 +358,41 @@ def highway_exp_setup(sim_params=None, # set default to no traffic lights traffic_lights = TrafficLightParams() + flow_params = dict( + # name of the experiment + exp_tag="RingRoadTest", + + # name of the flow environment the experiment is running on + env_name=AccelEnv, + + # name of the network class the experiment is running on + network=HighwayNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=sim_params, + + # environment related parameters (see flow.core.params.EnvParams) + env=env_params, + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=net_params, + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=initial_config, + + # traffic lights to be introduced to specific nodes (see + # flow.core.params.TrafficLightParams) + tls=traffic_lights, + ) + # create the network network = HighwayNetwork( name="RingRoadTest", @@ -299,7 +408,7 @@ def highway_exp_setup(sim_params=None, # reset the environment env.reset() - return env, network + return env, network, flow_params def traffic_light_grid_mxn_exp_setup(row_num=1, @@ -409,6 +518,41 @@ def traffic_light_grid_mxn_exp_setup(row_num=1, initial_config = InitialConfig( spacing="custom", additional_params={"enter_speed": 30}) + flow_params = dict( + # name of the experiment + exp_tag="Grid1x1Test", + + # name of the flow environment the experiment is running on + env_name=TrafficLightGridTestEnv, + + # name of the network class the experiment is running on + network=TrafficLightGridNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=sim_params, + + # environment related parameters (see flow.core.params.EnvParams) + env=env_params, + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=net_params, + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=initial_config, + + # traffic lights to be introduced to specific nodes (see + # flow.core.params.TrafficLightParams) + tls=tl_logic + ) + # create the network network = TrafficLightGridNetwork( name="Grid1x1Test", @@ -424,7 +568,7 @@ def traffic_light_grid_mxn_exp_setup(row_num=1, # reset the environment env.reset() - return env, network + return env, network, flow_params def variable_lanes_exp_setup(sim_params=None, @@ -505,6 +649,41 @@ def variable_lanes_exp_setup(sim_params=None, # set default to no traffic lights traffic_lights = TrafficLightParams() + flow_params = dict( + # name of the experiment + exp_tag="VariableLaneRingRoadTest", + + # name of the flow environment the experiment is running on + env_name=AccelEnv, + + # name of the network class the experiment is running on + network=VariableLanesNetwork, + + # simulator that is used by the experiment + simulator='traci', + + # sumo-related parameters (see flow.core.params.SumoParams) + sim=sim_params, + + # environment related parameters (see flow.core.params.EnvParams) + env=env_params, + # network-related parameters (see flow.core.params.NetParams and the + # network's documentation or ADDITIONAL_NET_PARAMS component) + net=net_params, + + # vehicles to be placed in the network at the start of a rollout (see + # flow.core.params.VehicleParams) + veh=vehicles, + + # parameters specifying the positioning of vehicles upon initialization/ + # reset (see flow.core.params.InitialConfig) + initial=initial_config, + + # traffic lights to be introduced to specific nodes (see + # flow.core.params.TrafficLightParams) + tls=traffic_lights, + ) + # create the network network = VariableLanesNetwork( name="VariableLaneRingRoadTest", @@ -520,7 +699,7 @@ def variable_lanes_exp_setup(sim_params=None, # reset the environment env.reset() - return env, network + return env, network, flow_params class VariableLanesNetwork(RingNetwork): diff --git a/tests/slow_tests/test_benchmarks.py b/tests/slow_tests/test_benchmarks.py index be59ddcf3..c119d4bd8 100644 --- a/tests/slow_tests/test_benchmarks.py +++ b/tests/slow_tests/test_benchmarks.py @@ -34,6 +34,11 @@ from flow.benchmarks.stable_baselines.trpo_runner import parse_args from flow.benchmarks.stable_baselines.trpo_runner import save_model +import random + +# This removes the randomness in this test +random.seed(a=10) + N_CPUS = 1 ray.init(num_cpus=N_CPUS) diff --git a/tests/stress_tests/stress_test_start.py b/tests/stress_tests/stress_test_start.py index 65702a121..ce457c1ee 100644 --- a/tests/stress_tests/stress_test_start.py +++ b/tests/stress_tests/stress_test_start.py @@ -28,13 +28,13 @@ def start(): initial_config = InitialConfig(bunching=20) network = RingNetwork( - name="sugiyama", + name="ring", vehicles=vehicles, net_params=net_params, initial_config=initial_config) env = AccelEnv(env_params, sim_params, network) - env._close() + env.close() ray.init() diff --git a/tutorials/README.md b/tutorials/README.md index a9845a686..6b4823068 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -20,10 +20,10 @@ cd /tutorials jupyter-notebook ``` -Instructions are written in each file. To do each exercise, first run all of +Instructions are written in each file. To do each tutorial, first run all of the cells in the Jupyter notebook. Then modify the ones that need to be modified in order to prevent any exceptions from being raised. Throughout these -exercises, you may find the +tutorials, you may find the [Flow documentation](https://flow.readthedocs.io/en/latest/) helpful. > **Common error:** if, when running a notebook, you run into an error of the form @@ -45,7 +45,7 @@ exercises, you may find the > ``` > (`cd /` is to make sure that `flow` is not in the folder you run `python` from). Then, in the Python interface that opens, run `import flow`. If you get an `ImportError`, this means you haven't installed Flow in your environment. Go back to the [installation instructions](https://flow.readthedocs.io/en/latest/flow_setup.html), especially the part where you do `pip install -e .` after having done `source activate flow`. -The content of each exercise is as follows: +The content of each tutorial is as follows: **Tutorial 0:** High-level introduction to Flow. @@ -56,7 +56,7 @@ The content of each exercise is as follows: **Tutorial 3:** Running RLlib experiments for mixed-autonomy traffic. **Tutorial 4:** Saving and visualizing resuls from non-RL simulations and -testing simulations in the presence of an RLlib/rllab agent. +testing simulations in the presence of an RLlib agent. **Tutorial 5:** Creating custom networks. diff --git a/tutorials/data/trained_ring/checkpoint_100/checkpoint-100 b/tutorials/data/trained_ring/checkpoint_100/checkpoint-100 new file mode 100644 index 000000000..9fcf839ed Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_100/checkpoint-100 differ diff --git a/tutorials/data/trained_ring/checkpoint_100/checkpoint-100.tune_metadata b/tutorials/data/trained_ring/checkpoint_100/checkpoint-100.tune_metadata new file mode 100644 index 000000000..d715df3e4 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_100/checkpoint-100.tune_metadata differ diff --git a/tutorials/data/trained_ring/checkpoint_120/checkpoint-120 b/tutorials/data/trained_ring/checkpoint_120/checkpoint-120 new file mode 100644 index 000000000..f7fe9d76f Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_120/checkpoint-120 differ diff --git a/tutorials/data/trained_ring/checkpoint_120/checkpoint-120.tune_metadata b/tutorials/data/trained_ring/checkpoint_120/checkpoint-120.tune_metadata new file mode 100644 index 000000000..a6d2c2c80 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_120/checkpoint-120.tune_metadata differ diff --git a/tutorials/data/trained_ring/checkpoint_140/checkpoint-140 b/tutorials/data/trained_ring/checkpoint_140/checkpoint-140 new file mode 100644 index 000000000..0556862b2 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_140/checkpoint-140 differ diff --git a/tutorials/data/trained_ring/checkpoint_140/checkpoint-140.tune_metadata b/tutorials/data/trained_ring/checkpoint_140/checkpoint-140.tune_metadata new file mode 100644 index 000000000..adcdc3630 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_140/checkpoint-140.tune_metadata differ diff --git a/tutorials/data/trained_ring/checkpoint_160/checkpoint-160 b/tutorials/data/trained_ring/checkpoint_160/checkpoint-160 new file mode 100644 index 000000000..9526f5f2d Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_160/checkpoint-160 differ diff --git a/tutorials/data/trained_ring/checkpoint_160/checkpoint-160.tune_metadata b/tutorials/data/trained_ring/checkpoint_160/checkpoint-160.tune_metadata new file mode 100644 index 000000000..29b451459 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_160/checkpoint-160.tune_metadata differ diff --git a/tutorials/data/trained_ring/checkpoint_180/checkpoint-180 b/tutorials/data/trained_ring/checkpoint_180/checkpoint-180 new file mode 100644 index 000000000..5df269f56 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_180/checkpoint-180 differ diff --git a/tutorials/data/trained_ring/checkpoint_180/checkpoint-180.tune_metadata b/tutorials/data/trained_ring/checkpoint_180/checkpoint-180.tune_metadata new file mode 100644 index 000000000..560a24ff8 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_180/checkpoint-180.tune_metadata differ diff --git a/tutorials/data/trained_ring/checkpoint_20/checkpoint-20 b/tutorials/data/trained_ring/checkpoint_20/checkpoint-20 new file mode 100644 index 000000000..971b7a38c Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_20/checkpoint-20 differ diff --git a/tutorials/data/trained_ring/checkpoint_20/checkpoint-20.tune_metadata b/tutorials/data/trained_ring/checkpoint_20/checkpoint-20.tune_metadata new file mode 100644 index 000000000..d47f0a639 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_20/checkpoint-20.tune_metadata differ diff --git a/tutorials/data/trained_ring/checkpoint_200/checkpoint-200 b/tutorials/data/trained_ring/checkpoint_200/checkpoint-200 new file mode 100644 index 000000000..6fc1e6058 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_200/checkpoint-200 differ diff --git a/tutorials/data/trained_ring/checkpoint_200/checkpoint-200.tune_metadata b/tutorials/data/trained_ring/checkpoint_200/checkpoint-200.tune_metadata new file mode 100644 index 000000000..131dbb7a6 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_200/checkpoint-200.tune_metadata differ diff --git a/tutorials/data/trained_ring/checkpoint_40/checkpoint-40 b/tutorials/data/trained_ring/checkpoint_40/checkpoint-40 new file mode 100644 index 000000000..e32de7867 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_40/checkpoint-40 differ diff --git a/tutorials/data/trained_ring/checkpoint_40/checkpoint-40.tune_metadata b/tutorials/data/trained_ring/checkpoint_40/checkpoint-40.tune_metadata new file mode 100644 index 000000000..78f8065fb Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_40/checkpoint-40.tune_metadata differ diff --git a/tutorials/data/trained_ring/checkpoint_60/checkpoint-60 b/tutorials/data/trained_ring/checkpoint_60/checkpoint-60 new file mode 100644 index 000000000..f4b9ea4d7 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_60/checkpoint-60 differ diff --git a/tutorials/data/trained_ring/checkpoint_60/checkpoint-60.tune_metadata b/tutorials/data/trained_ring/checkpoint_60/checkpoint-60.tune_metadata new file mode 100644 index 000000000..0fd02740e Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_60/checkpoint-60.tune_metadata differ diff --git a/tutorials/data/trained_ring/checkpoint_80/checkpoint-80 b/tutorials/data/trained_ring/checkpoint_80/checkpoint-80 new file mode 100644 index 000000000..ca8554ee4 Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_80/checkpoint-80 differ diff --git a/tutorials/data/trained_ring/checkpoint_80/checkpoint-80.tune_metadata b/tutorials/data/trained_ring/checkpoint_80/checkpoint-80.tune_metadata new file mode 100644 index 000000000..6602adafb Binary files /dev/null and b/tutorials/data/trained_ring/checkpoint_80/checkpoint-80.tune_metadata differ diff --git a/tutorials/data/trained_ring/params.json b/tutorials/data/trained_ring/params.json new file mode 100644 index 000000000..b0da309ef --- /dev/null +++ b/tutorials/data/trained_ring/params.json @@ -0,0 +1,125 @@ +{ + "batch_mode": "truncate_episodes", + "callbacks": { + "on_episode_end": null, + "on_episode_start": null, + "on_episode_step": null, + "on_postprocess_traj": null, + "on_sample_end": null, + "on_train_result": null + }, + "clip_actions": false, + "clip_param": 0.3, + "clip_rewards": null, + "collect_metrics_timeout": 180, + "compress_observations": false, + "custom_resources_per_worker": {}, + "entropy_coeff": 0.0, + "entropy_coeff_schedule": null, + "env": "WaveAttenuationPOEnv-v0", + "env_config": { + "flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", + "run": "PPO" + }, + "evaluation_config": {}, + "evaluation_interval": null, + "evaluation_num_episodes": 10, + "gamma": 0.999, + "grad_clip": null, + "horizon": 3000, + "ignore_worker_failures": false, + "input": "sampler", + "input_evaluation": [ + "is", + "wis" + ], + "kl_coeff": 0.2, + "kl_target": 0.02, + "lambda": 0.97, + "local_tf_session_args": { + "inter_op_parallelism_threads": 8, + "intra_op_parallelism_threads": 8 + }, + "log_level": "INFO", + "log_sys_usage": true, + "lr": 5e-05, + "lr_schedule": null, + "metrics_smoothing_episodes": 100, + "min_iter_time_s": 0, + "model": { + "conv_activation": "relu", + "conv_filters": null, + "custom_model": null, + "custom_options": {}, + "custom_preprocessor": null, + "dim": 84, + "fcnet_activation": "tanh", + "fcnet_hiddens": [ + 3, + 3 + ], + "framestack": true, + "free_log_std": false, + "grayscale": false, + "lstm_cell_size": 256, + "lstm_use_prev_action_reward": false, + "max_seq_len": 20, + "no_final_linear": false, + "state_shape": null, + "use_lstm": false, + "vf_share_layers": true, + "zero_mean": true + }, + "monitor": false, + "multiagent": { + "policies": {}, + "policies_to_train": null, + "policy_mapping_fn": null + }, + "num_cpus_for_driver": 1, + "num_cpus_per_worker": 1, + "num_envs_per_worker": 1, + "num_gpus": 0, + "num_gpus_per_worker": 0, + "num_sgd_iter": 10, + "num_workers": 2, + "observation_filter": "NoFilter", + "optimizer": {}, + "output": null, + "output_compress_columns": [ + "obs", + "new_obs" + ], + "output_max_file_size": 67108864, + "postprocess_inputs": false, + "preprocessor_pref": "deepmind", + "remote_env_batch_wait_ms": 0, + "remote_worker_envs": false, + "sample_async": false, + "sample_batch_size": 200, + "seed": null, + "sgd_minibatch_size": 128, + "shuffle_buffer_size": 0, + "shuffle_sequences": true, + "simple_optimizer": false, + "soft_horizon": false, + "synchronize_filters": true, + "tf_session_args": { + "allow_soft_placement": true, + "device_count": { + "CPU": 1 + }, + "gpu_options": { + "allow_growth": true + }, + "inter_op_parallelism_threads": 2, + "intra_op_parallelism_threads": 2, + "log_device_placement": false + }, + "timesteps_per_iteration": 0, + "train_batch_size": 60000, + "use_gae": true, + "vf_clip_param": 10.0, + "vf_loss_coeff": 1.0, + "vf_share_layers": false +} \ No newline at end of file diff --git a/tutorials/data/trained_ring/params.pkl b/tutorials/data/trained_ring/params.pkl new file mode 100644 index 000000000..d48ee0658 Binary files /dev/null and b/tutorials/data/trained_ring/params.pkl differ diff --git a/tutorials/data/trained_ring/progress.csv b/tutorials/data/trained_ring/progress.csv new file mode 100644 index 000000000..19bfefd44 --- /dev/null +++ b/tutorials/data/trained_ring/progress.csv @@ -0,0 +1,201 @@ +episode_reward_max,episode_reward_min,episode_reward_mean,episode_len_mean,episodes_this_iter,timesteps_this_iter,done,timesteps_total,episodes_total,training_iteration,experiment_id,date,timestamp,time_this_iter_s,time_total_s,pid,hostname,node_ip,time_since_restore,timesteps_since_restore,iterations_since_restore,num_healthy_workers,trial_id,sampler_perf/mean_env_wait_ms,sampler_perf/mean_processing_ms,sampler_perf/mean_inference_ms,info/num_steps_trained,info/num_steps_sampled,info/sample_time_ms,info/load_time_ms,info/grad_time_ms,info/update_time_ms,perf/cpu_util_percent,perf/ram_util_percent,info/learner/default_policy/cur_kl_coeff,info/learner/default_policy/cur_lr,info/learner/default_policy/total_loss,info/learner/default_policy/policy_loss,info/learner/default_policy/vf_loss,info/learner/default_policy/vf_explained_var,info/learner/default_policy/kl,info/learner/default_policy/entropy,info/learner/default_policy/entropy_coeff +-7025.808506814366,-9261.36558597124,-8112.181357017001,3000.0,20,60000,False,60000,20,1,fec93e2e40d94966abd856ec126a6b85,2019-10-10_20-18-55,1570763935,228.52161312103271,228.52161312103271,21927,ellie,10.0.0.158,228.52161312103271,60000,1,2,4dce3444,5.088170867479468,1.4314549290566863,0.7895048544933859,59904,60000,222168.726,70.708,5654.924,559.798,52.21559633027523,71.73363914373088,0.20000000298023224,4.999999873689376e-05,5841.9599609375,-0.010422835126519203,5841.9677734375,-6.949774251552299e-05,0.011039670556783676,1.3680330514907837,0.0 +-5906.2140718487135,-9261.36558597124,-7658.758736087065,3000.0,20,60000,False,120000,40,2,fec93e2e40d94966abd856ec126a6b85,2019-10-10_20-22-36,1570764156,221.10058522224426,449.622198343277,21927,ellie,10.0.0.158,449.622198343277,120000,2,2,4dce3444,5.0538187367921905,1.4290795645209107,0.7888463681015899,119808,120000,219066.136,40.976,5381.374,282.127,51.92380952380953,71.6184126984127,0.20000000298023224,4.999999873689376e-05,4593.02490234375,-0.01005244255065918,4593.033203125,-4.692286529461853e-05,0.010486019775271416,1.318793773651123,0.0 +-5013.736690034951,-9261.36558597124,-7143.2109891870605,3000.0,20,60000,False,180000,60,3,fec93e2e40d94966abd856ec126a6b85,2019-10-10_20-26-18,1570764378,222.16285920143127,671.7850575447083,21927,ellie,10.0.0.158,671.7850575447083,180000,3,2,4dce3444,5.039747437323658,1.428160692276646,0.788035512343128,179712,180000,218389.836,31.292,5285.999,189.445,51.99432176656152,71.66403785488959,0.20000000298023224,4.999999873689376e-05,3316.42529296875,-0.010575403459370136,3316.433837890625,-5.873668487765826e-05,0.011085903272032738,1.2667220830917358,0.0 +-4456.1761653801295,-9261.36558597124,-6669.404688050863,3000.0,20,60000,False,240000,80,4,fec93e2e40d94966abd856ec126a6b85,2019-10-10_20-30-03,1570764603,225.03357934951782,896.8186368942261,21927,ellie,10.0.0.158,896.8186368942261,240000,4,2,4dce3444,5.030053462651056,1.4308519019395238,0.7870829591072147,239616,240000,218664.778,28.448,5340.407,143.086,51.666043613707174,71.84859813084113,0.20000000298023224,4.999999873689376e-05,2439.493896484375,-0.011184969916939735,2439.502685546875,-2.6020992663688958e-05,0.011975754983723164,1.2135356664657593,0.0 +-3487.5620037295853,-9261.36558597124,-6321.31384678577,3000.0,20,60000,False,300000,100,5,fec93e2e40d94966abd856ec126a6b85,2019-10-10_20-34-12,1570764852,248.96133828163147,1145.7799751758575,21927,ellie,10.0.0.158,1145.7799751758575,300000,5,2,4dce3444,5.044428040388933,1.4386925304952547,0.7907806502716475,299520,300000,223430.825,25.437,5558.887,115.179,58.12845070422535,61.5149295774648,0.20000000298023224,4.999999873689376e-05,2172.185302734375,-0.011577743105590343,2172.194091796875,-7.91952152212616e-06,0.012583091855049133,1.164360523223877,0.0 +-2854.9588710832304,-8096.150249372029,-5476.13577015095,3000.0,20,60000,False,360000,120,6,fec93e2e40d94966abd856ec126a6b85,2019-10-10_20-38-35,1570765115,262.6561372280121,1408.4361124038696,21927,ellie,10.0.0.158,1408.4361124038696,360000,6,2,4dce3444,5.073097671189653,1.4529774357459888,0.7989281480451976,359424,360000,228890.611,23.431,5704.099,96.898,59.342666666666666,45.55626666666667,0.20000000298023224,4.999999873689376e-05,1362.3341064453125,-0.012156374752521515,1362.3438720703125,-2.1534087863983586e-06,0.012464698404073715,1.1054127216339111,0.0 +-2221.00948676975,-7149.132676521768,-4656.074478866747,3000.0,20,60000,False,420000,140,7,fec93e2e40d94966abd856ec126a6b85,2019-10-10_20-43-30,1570765410,294.7735686302185,1703.2096810340881,21927,ellie,10.0.0.158,1703.2096810340881,420000,7,2,4dce3444,5.153275956053091,1.4770714013878166,0.8140585646975877,419328,420000,236927.77,22.291,6257.975,83.76,65.39642857142857,49.371904761904766,0.20000000298023224,4.999999873689376e-05,870.5482788085938,-0.011359051801264286,870.5573120117188,-7.463316364919592e-07,0.011871282942593098,1.0494786500930786,0.0 +-1452.2763886998616,-6322.268008276782,-3933.3365328160976,3000.0,20,60000,False,480000,160,8,fec93e2e40d94966abd856ec126a6b85,2019-10-10_20-49-08,1570765748,338.6552450656891,2041.8649260997772,21927,ellie,10.0.0.158,2041.8649260997772,480000,8,2,4dce3444,5.290933627345668,1.5120344816730351,0.8402113955101816,479232,480000,248612.429,21.439,6501.322,73.941,69.82111801242236,53.56438923395445,0.20000000298023224,4.999999873689376e-05,597.2427978515625,-0.012078477069735527,597.2522583007812,-4.884269628746551e-07,0.013157332316040993,0.9939395785331726,0.0 +-1118.3189766642274,-6124.9506407198605,-3307.968411093388,3000.0,20,60000,False,540000,180,9,fec93e2e40d94966abd856ec126a6b85,2019-10-10_20-55-02,1570766102,353.79422402381897,2395.659150123596,21927,ellie,10.0.0.158,2395.659150123596,540000,9,2,4dce3444,5.480940687804025,1.55489631491183,0.8770157807184243,539136,540000,259234.077,20.498,6839.633,67.039,71.38452380952381,55.07023809523809,0.20000000298023224,4.999999873689376e-05,409.6300048828125,-0.013440164737403393,409.6407165527344,-6.151505260731938e-08,0.0134345144033432,0.9395018815994263,0.0 +-810.4557983671579,-4998.431657411379,-2668.5023948431453,3000.0,20,60000,False,600000,200,10,fec93e2e40d94966abd856ec126a6b85,2019-10-10_20-59-46,1570766386,283.7291479110718,2679.388298034668,21927,ellie,10.0.0.158,2679.388298034668,600000,10,2,4dce3444,5.659735332467301,1.5931577852227674,0.9113241213176999,599040,600000,260963.121,19.964,6872.463,60.652,61.58444444444444,47.998024691358026,0.20000000298023224,4.999999873689376e-05,290.8235778808594,-0.011462359689176083,290.83251953125,-2.2670143096092943e-08,0.01251077651977539,0.8885784149169922,0.0 +-93.82992271840153,-4293.155475649472,-2089.294018988054,3000.0,20,60000,False,660000,220,11,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-04-34,1570766674,287.5162422657013,2966.9045403003693,21927,ellie,10.0.0.158,2966.9045403003693,660000,11,2,4dce3444,5.821826827737394,1.6264132822510564,0.9423207702497534,658944,660000,266770.744,14.156,7030.347,5.173,61.77024390243902,47.73658536585366,0.20000000298023224,4.999999873689376e-05,114.49359130859375,-0.011447830125689507,114.50259399414062,-2.2542781863421624e-08,0.012185333296656609,0.8246817588806152,0.0 +108.44067851910259,-3942.7840997588282,-1632.5435440635117,3000.0,20,60000,False,720000,240,12,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-09-19,1570766959,285.7416036128998,3252.646143913269,21927,ellie,10.0.0.158,3252.646143913269,720000,12,2,4dce3444,5.953380432188706,1.6516868168672698,0.9678841027984157,718848,720000,273030.331,14.465,7234.085,5.306,61.30098039215686,48.312254901960785,0.20000000298023224,4.999999873689376e-05,85.70304870605469,-0.011727341450750828,85.71227264404297,-1.1080350859060673e-08,0.012510515749454498,0.7761985659599304,0.0 +673.5586714703026,-2839.3240533315893,-1219.8554623415228,3000.0,20,60000,False,780000,260,13,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-14-04,1570767244,285.16099739074707,3537.807141304016,21927,ellie,10.0.0.158,3537.807141304016,780000,13,2,4dce3444,6.034394240623488,1.6665883136925579,0.9837252258844218,778752,780000,279125.462,14.509,7438.321,5.279,60.53029556650246,49.21108374384237,0.20000000298023224,4.999999873689376e-05,57.19596862792969,-0.010404279455542564,57.20412826538086,-1.5283243115504774e-08,0.011229820549488068,0.7205685377120972,0.0 +799.0577524733043,-2603.272183032721,-807.7392234128113,3000.0,20,60000,False,840000,280,14,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-18-54,1570767534,289.7724847793579,3827.579626083374,21927,ellie,10.0.0.158,3827.579626083374,840000,14,2,4dce3444,6.071445044177026,1.6721416835638423,0.9903971013706158,838656,840000,285426.019,13.711,7612.328,5.314,62.149033816425124,50.398792270531395,0.20000000298023224,4.999999873689376e-05,39.828609466552734,-0.009519231505692005,39.83600997924805,0.0,0.010611079633235931,0.6647995114326477,0.0 +1081.353797705279,-2141.3476536829326,-446.89969052887267,3000.0,20,60000,False,900000,300,15,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-23-40,1570767820,285.5707211494446,4113.150347232819,21927,ellie,10.0.0.158,4113.150347232819,900000,15,2,4dce3444,6.10383208302859,1.676796884919757,0.9960639894286928,898560,900000,289019.073,13.447,7680.257,5.651,61.191891891891885,50.94766584766585,0.20000000298023224,4.999999873689376e-05,41.71358871459961,-0.008507564663887024,41.720245361328125,-2.292486422916795e-09,0.009255404584109783,0.6216187477111816,0.0 +1101.8440176363677,-2141.3476536829326,-146.5685228869649,3000.0,20,60000,False,960000,320,16,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-28-27,1570768107,287.00454568862915,4400.154892921448,21927,ellie,10.0.0.158,4400.154892921448,960000,16,2,4dce3444,6.131669891983649,1.6804058860614495,1.00061880971467,958464,960000,291389.424,13.496,7744.557,5.788,60.963080684596584,51.38117359413202,0.10000000149011612,4.999999873689376e-05,46.670047760009766,-0.008123466745018959,46.67705535888672,-5.221774568298088e-09,0.011164167895913124,0.5574238896369934,0.0 +1515.6486623734095,-1638.0720599202787,129.62546895883378,3000.0,20,60000,False,1020000,340,17,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-33-14,1570768394,286.7824501991272,4686.937343120575,21927,ellie,10.0.0.158,4686.937343120575,1020000,17,2,4dce3444,6.156121243797343,1.683453019331702,1.004437954877005,1018368,1020000,290834.267,13.311,7501.34,5.64,61.199999999999996,49.88068459657701,0.10000000149011612,4.999999873689376e-05,53.124549865722656,-0.0074589853174984455,53.13093566894531,-2.1651260784238957e-09,0.010762547142803669,0.5129155516624451,0.0 +1660.846489412573,-1034.344492390856,361.5233325313951,3000.0,20,60000,False,1080000,360,18,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-38-10,1570768690,296.4012680053711,4983.338611125946,21927,ellie,10.0.0.158,4983.338611125946,1080000,18,2,4dce3444,6.181265086724607,1.686685512821029,1.008347753943032,1078272,1080000,286661.587,13.458,7448.908,5.653,63.65886524822695,49.63569739952718,0.10000000149011612,4.999999873689376e-05,66.31993103027344,-0.006132551934570074,66.32515716552734,-8.787864658188482e-09,0.009022381156682968,0.4667024612426758,0.0 +1744.5129197469237,-680.3310403663952,543.0221892787458,3000.0,20,60000,False,1140000,380,19,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-43-13,1570768993,303.31166338920593,5286.650274515152,21927,ellie,10.0.0.158,5286.650274515152,1140000,19,2,4dce3444,6.2068261160707685,1.689920375901492,1.0126095765594965,1138176,1140000,281797.399,13.429,7264.563,5.68,63.960046189376435,50.69214780600462,0.05000000074505806,4.999999873689376e-05,91.78323364257812,-0.004493150394409895,91.78736114501953,3.1840088343670914e-09,0.007295538205653429,0.42863214015960693,0.0 +1857.1969157003678,-533.1407552633883,760.7714926088873,3000.0,20,60000,False,1200000,400,20,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-48-17,1570769297,303.47370505332947,5590.123979568481,21927,ellie,10.0.0.158,5590.123979568481,1200000,20,2,4dce3444,6.234304177337799,1.6933028682206401,1.0172869348789584,1198080,1200000,283712.481,13.581,7323.056,6.039,63.80925925925926,51.37939814814815,0.02500000037252903,4.999999873689376e-05,132.9980010986328,-0.003935374319553375,133.0017547607422,-1.0188828669654981e-09,0.007829303853213787,0.3700675666332245,0.0 +1885.0495048915807,-533.1407552633883,883.5141535219582,3000.0,20,60000,False,1260000,420,21,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-52-18,1570769538,241.17777752876282,5831.301757097244,21927,ellie,10.0.0.158,5831.301757097244,1260000,21,2,4dce3444,6.249043481372505,1.6943665847044433,1.019800933823224,1257984,1260000,279290.455,13.348,7111.788,6.362,54.82470930232558,41.23808139534884,0.012500000186264515,4.999999873689376e-05,130.7291717529297,-0.00409942539408803,130.733154296875,-2.801927800888393e-09,0.00828665029257536,0.3369026482105255,0.0 +2018.8200815290775,-106.56830443114728,1018.4786740859113,3000.0,20,60000,False,1320000,440,22,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-56-10,1570769770,231.86295461654663,6063.164711713791,21927,ellie,10.0.0.158,6063.164711713791,1320000,22,2,4dce3444,6.25072077536659,1.6932793552776686,1.01977435153495,1317888,1320000,274102.656,12.896,6912.674,6.229,55.576132930513594,38.05015105740182,0.0062500000931322575,4.999999873689376e-05,144.9283905029297,-0.0042753373272717,144.93260192871094,-3.1840088343670914e-09,0.008732318878173828,0.2904764413833618,0.0 +2018.8200815290775,-106.56830443114728,1121.6036093403736,3000.0,20,60000,False,1380000,460,23,fec93e2e40d94966abd856ec126a6b85,2019-10-10_21-59-57,1570769997,226.61901426315308,6289.783725976944,21927,ellie,10.0.0.158,6289.783725976944,1380000,23,2,4dce3444,6.236836122936435,1.689467396021979,1.0167814796001757,1377792,1380000,268413.283,12.698,6748.455,6.306,51.745061728395065,37.803395061728395,0.0031250000465661287,4.999999873689376e-05,143.2877197265625,-0.003045490710064769,143.29074096679688,-8.405783624709784e-09,0.006720947567373514,0.2587600648403168,0.0 +2171.2679848606563,-103.90599867112795,1208.9556858688563,3000.0,20,60000,False,1440000,480,24,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-03-42,1570770222,225.26621341705322,6515.049939393997,21927,ellie,10.0.0.158,6515.049939393997,1440000,24,2,4dce3444,6.207254563235238,1.6824385542798972,1.0105801029295105,1437696,1440000,262176.436,12.489,6535.291,6.315,51.45015576323988,37.77476635514018,0.0015625000232830644,4.999999873689376e-05,157.0971221923828,-0.003159629413858056,157.1002655029297,2.4198467674096946e-09,0.007727622054517269,0.22158615291118622,0.0 +2177.282689812078,-103.90599867112795,1241.8399876042317,3000.0,20,60000,False,1500000,500,25,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-07-41,1570770461,238.85392904281616,6753.903868436813,21927,ellie,10.0.0.158,6753.903868436813,1500000,25,2,4dce3444,6.1654521680222185,1.6733859633169788,1.001977321894367,1497600,1500000,257629.305,12.482,6411.156,6.104,55.407624633431084,40.25307917888563,0.0007812500116415322,4.999999873689376e-05,157.69520568847656,-0.0025367350317537785,157.69775390625,-1.2099233615003868e-08,0.00578154856339097,0.18547388911247253,0.0 +2346.6717922782086,-35.48750666518595,1336.7274591634734,3000.0,20,60000,False,1560000,520,26,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-11-27,1570770687,225.8318202495575,6979.735688686371,21927,ellie,10.0.0.158,6979.735688686371,1560000,26,2,4dce3444,6.124262021320062,1.664400394419175,0.9934427224858094,1557504,1560000,251716.412,12.087,6207.78,5.887,51.75993788819876,37.82329192546583,0.0003906250058207661,4.999999873689376e-05,221.3697509765625,-0.0030829834286123514,221.37283325195312,-1.273603555951297e-09,0.006834876723587513,0.1253022998571396,0.0 +2473.0046252112115,-35.48750666518595,1444.352684346824,3000.0,20,60000,False,1620000,540,27,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-15-17,1570770917,230.90782046318054,7210.643509149551,21927,ellie,10.0.0.158,7210.643509149551,1620000,27,2,4dce3444,6.086305165552326,1.6557962568319342,0.9855702026004194,1617408,1620000,246342.935,11.78,5994.347,6.009,54.23343465045593,37.77355623100304,0.00019531250291038305,4.999999873689376e-05,271.76629638671875,-0.001978727290406823,271.76824951171875,-6.240657324241283e-09,0.0051463027484714985,0.0797242596745491,0.0 +2473.0046252112115,-35.48750666518595,1501.5721173817913,3000.0,20,60000,False,1680000,560,28,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-19-01,1570771141,223.33587646484375,7433.979385614395,21927,ellie,10.0.0.158,7433.979385614395,1680000,28,2,4dce3444,6.050953980575351,1.647533473699927,0.9782470348249817,1677312,1680000,239298.803,11.091,5733.348,5.918,51.34420062695926,37.992163009404386,9.765625145519152e-05,4.999999873689376e-05,203.982177734375,-0.0018949565710499883,203.98406982421875,-5.858576290762585e-09,0.005937520414590836,0.07262831926345825,0.0 +2473.0046252112115,662.7724399754616,1553.8884773009734,3000.0,20,60000,False,1740000,580,29,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-22-44,1570771364,223.01602435112,7656.995409965515,21927,ellie,10.0.0.158,7656.995409965515,1740000,29,2,4dce3444,6.018071470675672,1.6397858567581012,0.9714509168392169,1737216,1740000,231535.327,10.841,5468.859,5.12,51.29150943396227,37.99559748427673,4.882812572759576e-05,4.999999873689376e-05,199.8420867919922,-0.0020953549537807703,199.8441619873047,-9.679387069638778e-09,0.005188964307308197,0.0404723696410656,0.0 +2573.934205801572,824.2240294089684,1659.2793911728618,3000.0,20,60000,False,1800000,600,30,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-26-32,1570771592,227.87344074249268,7884.868850708008,21927,ellie,10.0.0.158,7884.868850708008,1800000,30,2,4dce3444,5.985879629528855,1.6318744501938778,0.9647372745949525,1797120,1800000,224231.576,10.168,5214.341,4.914,51.66676923076923,39.36923076923076,2.441406286379788e-05,4.999999873689376e-05,296.41766357421875,-0.0018888604827225208,296.4195251464844,-3.222216804488198e-08,0.0052628349512815475,-0.018881741911172867,0.0 +2616.9848104271323,824.2240294089684,1717.053522849597,3000.0,20,60000,False,1860000,620,31,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-30-19,1570771819,227.49058842658997,8112.359439134598,21927,ellie,10.0.0.158,8112.359439134598,1860000,31,2,4dce3444,5.9560246038940825,1.6245303057639364,0.9585262139783197,1857024,1860000,222865.698,10.157,5211.954,4.464,51.446461538461534,39.654769230769226,1.220703143189894e-05,4.999999873689376e-05,308.9244384765625,-0.0007358700968325138,308.9251403808594,-1.7957809461677243e-08,0.003371907165274024,-0.0426129549741745,0.0 +2616.9848104271323,824.2240294089684,1769.2830190951656,3000.0,20,60000,False,1920000,640,32,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-34-07,1570772047,228.3018355369568,8340.661274671555,21927,ellie,10.0.0.158,8340.661274671555,1920000,32,2,4dce3444,5.927406970468294,1.6176372363879121,0.9526021235933171,1916928,1920000,222508.282,10.181,5213.161,4.482,51.368307692307695,39.601230769230774,6.10351571594947e-06,4.999999873689376e-05,350.0728454589844,-0.0010255386587232351,350.0738525390625,-1.630212587144797e-08,0.0036517020780593157,-0.08226603269577026,0.0 +2616.9848104271323,988.5700450267395,1848.0285179073728,3000.0,20,60000,False,1980000,660,33,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-37-55,1570772275,227.6798312664032,8568.341105937958,21927,ellie,10.0.0.158,8568.341105937958,1980000,33,2,4dce3444,5.901369999157236,1.6115178438951623,0.9471200859892064,1976832,1980000,222653.758,10.167,5173.896,4.438,51.74861538461539,39.59938461538462,3.051757857974735e-06,4.999999873689376e-05,301.1153259277344,-0.0014899128582328558,301.1168212890625,-1.897669221762044e-08,0.005265341140329838,-0.1068715825676918,0.0 +2693.5456715325713,988.5700450267395,1911.9438265035753,3000.0,20,60000,False,2040000,680,34,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-41-41,1570772501,225.9329686164856,8794.274074554443,21927,ellie,10.0.0.158,8794.274074554443,2040000,34,2,4dce3444,5.877489217012647,1.605949389614291,0.9420209901324407,2036736,2040000,222723.351,10.157,5170.939,4.437,51.452173913043474,39.59937888198758,1.5258789289873675e-06,4.999999873689376e-05,293.3316650390625,-0.0008449004380963743,293.3324890136719,-5.349134912790987e-09,0.004025230649858713,-0.11745946854352951,0.0 +2693.5456715325713,988.5700450267395,1936.2141707487754,3000.0,20,60000,False,2100000,700,35,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-45-28,1570772728,226.53039646148682,9020.80447101593,21927,ellie,10.0.0.158,9020.80447101593,2100000,35,2,4dce3444,5.854872266222085,1.6008661423817119,0.9372015370668023,2096640,2100000,221565.32,10.107,5096.784,4.341,51.77368421052632,39.60061919504644,7.629394644936838e-07,4.999999873689376e-05,320.0366516113281,-0.0009304272243753076,320.0376281738281,1.528324244937096e-09,0.0032853041775524616,-0.15430699288845062,0.0 +2693.5456715325713,988.5700450267395,1930.5465885275719,3000.0,20,60000,False,2160000,720,36,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-49-14,1570772954,226.03132367134094,9246.835794687271,21927,ellie,10.0.0.158,9246.835794687271,2160000,36,2,4dce3444,5.833342924196859,1.5960547506565224,0.9326516376453324,2156544,2160000,221577.781,10.131,5104.264,4.325,51.382972136222925,39.59969040247679,3.814697322468419e-07,4.999999873689376e-05,292.1423034667969,-0.0013726623728871346,292.1437072753906,-2.2160701718121345e-08,0.004224446602165699,-0.16808035969734192,0.0 +2776.6405679157083,1093.7900345984647,1945.0163757393439,3000.0,20,60000,False,2220000,740,37,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-53-02,1570773182,228.20357489585876,9475.03936958313,21927,ellie,10.0.0.158,9475.03936958313,2220000,37,2,4dce3444,5.812983189318344,1.591521016067461,0.9283693052454841,2216448,2220000,221296.094,10.097,5115.562,4.328,51.33046153846154,39.60092307692309,1.9073486612342094e-07,4.999999873689376e-05,367.0290222167969,-0.0010790476808324456,367.03009033203125,-2.6363592198208607e-08,0.006083440966904163,-0.20631200075149536,0.0 +2776.6405679157083,1156.7972621860602,1950.0848381205637,3000.0,20,60000,False,2280000,760,38,fec93e2e40d94966abd856ec126a6b85,2019-10-10_22-56-49,1570773409,226.96341514587402,9702.002784729004,21927,ellie,10.0.0.158,9702.002784729004,2280000,38,2,4dce3444,5.793628959826196,1.5871995447056495,0.9243322535528041,2276352,2280000,221657.953,10.129,5116.435,4.355,51.65370370370369,39.6,9.536743306171047e-08,4.999999873689376e-05,312.8647155761719,-0.0005480647087097168,312.86529541015625,-4.610444648278644e-08,0.0033533277455717325,-0.21347717940807343,0.0 +2776.6405679157083,1037.897900478011,1962.2869825974358,3000.0,20,60000,False,2340000,780,39,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-00-35,1570773635,226.29635906219482,9928.299143791199,21927,ellie,10.0.0.158,9928.299143791199,2340000,39,2,4dce3444,5.775375571550589,1.5831787139466353,0.9205203030347175,2336256,2340000,221979.38,10.173,5122.914,4.401,51.54272445820433,39.60092879256967,4.7683716530855236e-08,4.999999873689376e-05,312.23712158203125,-0.0005565821775235236,312.2376403808594,-4.1137393935741784e-08,0.0023237953428179026,-0.22383910417556763,0.0 +2776.6405679157083,1037.897900478011,1974.5461355925602,3000.0,20,60000,False,2400000,800,40,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-04-23,1570773863,228.18473315238953,10156.483876943588,21927,ellie,10.0.0.158,10156.483876943588,2400000,40,2,4dce3444,5.758149660119385,1.5793229004773917,0.9169234915072964,2396160,2400000,222022.82,10.228,5110.49,4.407,51.333742331288356,39.612576687116565,2.3841858265427618e-08,4.999999873689376e-05,338.8609313964844,-0.0010798104340210557,338.86199951171875,-3.718922414464032e-08,0.003974047023802996,-0.26470670104026794,0.0 +2784.3382585599466,1037.897900478011,2034.1331183414325,3000.0,20,60000,False,2460000,820,41,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-08-11,1570774091,227.9981415271759,10384.482018470764,21927,ellie,10.0.0.158,10384.482018470764,2460000,41,2,4dce3444,5.742247119261552,1.575586036721535,0.913545587494423,2456064,2460000,222069.957,10.194,5114.018,4.496,51.42553846153847,39.63876923076923,1.1920929132713809e-08,4.999999873689376e-05,372.523193359375,-0.00048392629832960665,372.5236511230469,-3.324105080082518e-08,0.003001442179083824,-0.2981806993484497,0.0 +2784.3382585599466,909.305565800551,1978.7946372907566,3000.0,20,60000,False,2520000,840,42,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-11-55,1570774315,223.84502983093262,10608.327048301697,21927,ellie,10.0.0.158,10608.327048301697,2520000,42,2,4dce3444,5.7267659215024,1.5718495197045461,0.9103305837742315,2515968,2520000,221636.299,10.175,5101.992,4.505,51.5,39.690282131661434,5.9604645663569045e-09,4.999999873689376e-05,276.86187744140625,-0.0002941251150332391,276.8622131347656,-3.094856637630983e-08,0.003100535599514842,-0.2838324010372162,0.0 +2784.3382585599466,909.305565800551,2004.5044007690408,3000.0,20,60000,False,2580000,860,43,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-15-45,1570774545,229.84651684761047,10838.173565149307,21927,ellie,10.0.0.158,10838.173565149307,2580000,43,2,4dce3444,5.712289092222843,1.5683120312660341,0.9073048101891489,2575872,2580000,221853.994,10.18,5100.811,4.514,53.3734756097561,39.767682926829266,2.9802322831784522e-09,4.999999873689376e-05,342.7833557128906,-0.0010411189869046211,342.784423828125,-2.610887150922281e-08,0.006138394586741924,-0.32215142250061035,0.0 +2784.3382585599466,909.305565800551,2001.5831379296658,3000.0,20,60000,False,2640000,880,44,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-19-32,1570774772,226.58823561668396,11064.761800765991,21927,ellie,10.0.0.158,11064.761800765991,2640000,44,2,4dce3444,5.698582902120918,1.564866482511378,0.9044546734094819,2635776,2640000,221920.955,10.225,5099.336,4.514,51.426934984520116,39.89938080495355,1.4901161415892261e-09,4.999999873689376e-05,296.57952880859375,-0.0004189646278973669,296.5799560546875,2.6745674563954935e-09,0.0030859194230288267,-0.31912147998809814,0.0 +2784.3382585599466,909.305565800551,1958.6506716824038,3000.0,20,60000,False,2700000,900,45,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-23-19,1570774999,227.1653265953064,11291.927127361298,21927,ellie,10.0.0.158,11291.927127361298,2700000,45,2,4dce3444,5.685303008469853,1.5616100196434957,0.9017471943642369,2695680,2700000,221985.268,10.197,5098.447,4.628,51.33302469135802,39.89907407407407,7.450580707946131e-10,4.999999873689376e-05,265.79290771484375,-0.0009432857623323798,265.7938232421875,-5.094414223805188e-09,0.002575733233243227,-0.3287334144115448,0.0 +2764.5460546960426,909.305565800551,1940.7206650363337,3000.0,20,60000,False,2760000,920,46,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-27-07,1570775227,228.3132302761078,11520.240357637405,21927,ellie,10.0.0.158,11520.240357637405,2760000,46,2,4dce3444,5.67261003747345,1.5586849210772256,0.899194261659875,2755584,2760000,222211.219,10.21,5100.666,4.646,51.72883435582822,39.899386503067475,3.7252903539730653e-10,4.999999873689376e-05,340.6348571777344,-0.00033470403286628425,340.63519287109375,-1.7830449339228949e-09,0.003041619434952736,-0.3805379569530487,0.0 +2764.5460546960426,1118.3477549045203,2003.4134713914696,3000.0,20,60000,False,2820000,940,47,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-30-54,1570775454,227.20234751701355,11747.442705154419,21927,ellie,10.0.0.158,11747.442705154419,2820000,47,2,4dce3444,5.6610115785514825,1.5559140257059776,0.8968104054655179,2815488,2820000,222113.488,10.191,5098.286,4.636,51.59814814814815,39.89938271604938,1.8626451769865326e-10,4.999999873689376e-05,369.5340576171875,-3.198018475814024e-06,369.5340576171875,-2.1651260784238957e-09,0.003405811032280326,-0.3966348469257355,0.0 +2763.541614495298,1118.3477549045203,1935.2097842175808,3000.0,20,60000,False,2880000,960,48,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-34-40,1570775680,225.27441596984863,11972.717121124268,21927,ellie,10.0.0.158,11972.717121124268,2880000,48,2,4dce3444,5.649386125975444,1.553293031933046,0.8945072540306853,2875392,2880000,221942.58,10.14,5100.424,4.62,51.63447204968945,39.9,9.313225884932663e-11,4.999999873689376e-05,235.3415069580078,-0.0004265991155989468,235.34194946289062,-6.622738357719982e-09,0.003912602551281452,-0.3681495189666748,0.0 +2763.541614495298,1118.3477549045203,1952.9976265574605,3000.0,20,60000,False,2940000,980,49,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-38-27,1570775907,227.061918258667,12199.779039382935,21927,ellie,10.0.0.158,12199.779039382935,2940000,49,2,4dce3444,5.638318252048253,1.5508499834906309,0.8922921938687176,2935296,2940000,222022.731,10.124,5096.844,4.629,51.55572755417957,39.89969040247678,4.6566129424663316e-11,4.999999873689376e-05,314.6124267578125,-0.00013308048073668033,314.6125183105469,-4.839693534819389e-09,0.003223870415240526,-0.40551745891571045,0.0 +2763.541614495298,1159.3331360048546,1989.8064041701018,3000.0,20,60000,False,3000000,1000,50,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-42-15,1570776135,227.88091659545898,12427.659955978394,21927,ellie,10.0.0.158,12427.659955978394,3000000,50,2,4dce3444,5.628015489307915,1.5484684826664992,0.8902062657092839,2995200,3000000,221989.03,10.1,5100.174,4.647,51.739570552147235,39.90153374233128,2.3283064712331658e-11,4.999999873689376e-05,319.1338806152344,-0.0006387312314473093,319.1344909667969,-4.0755314678619925e-09,0.002699590288102627,-0.42596644163131714,0.0 +2747.2897032528185,1142.1583378230064,1926.810640347007,3000.0,20,60000,False,3060000,1020,51,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-46-00,1570776360,224.83476877212524,12652.494724750519,21927,ellie,10.0.0.158,12652.494724750519,3060000,51,2,4dce3444,5.61774479317638,1.5460156789176551,0.8881574764052952,3055104,3060000,221677.828,10.137,5095.029,4.618,51.411874999999995,39.89999999999999,1.1641532356165829e-11,4.999999873689376e-05,244.47679138183594,-0.0007065013633109629,244.47750854492188,-2.1651260784238957e-09,0.0038327837828546762,-0.4098336696624756,0.0 +2747.2897032528185,849.5323429650005,1846.3953676391588,3000.0,20,60000,False,3120000,1040,52,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-49-44,1570776584,224.4415910243988,12876.936315774918,21927,ellie,10.0.0.158,12876.936315774918,3120000,52,2,4dce3444,5.607525936256994,1.5436799964025325,0.8861643324832422,3115008,3120000,221741.947,10.18,5090.579,4.619,51.7865625,38.92,5.8207661780829145e-12,4.999999873689376e-05,246.7576904296875,-0.000154356675921008,246.75784301757812,-1.1080350859060673e-08,0.0025222529657185078,-0.42039740085601807,0.0 +2747.2897032528185,849.5323429650005,1840.7385588912368,3000.0,20,60000,False,3180000,1060,53,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-53-27,1570776807,223.1104326248169,13100.046748399734,21927,ellie,10.0.0.158,13100.046748399734,3180000,53,2,4dce3444,5.597553581817157,1.541261210592515,0.8842086547485476,3174912,3180000,221078.755,10.153,5080.255,4.664,51.20501567398119,38.3037617554859,2.9103830890414573e-12,4.999999873689376e-05,231.25408935546875,-0.0010070799617096782,231.25509643554688,-3.8208107788761936e-09,0.006194713991135359,-0.42542222142219543,0.0 +2788.2348789393354,849.5323429650005,1830.1222636539665,3000.0,20,60000,False,3240000,1080,54,fec93e2e40d94966abd856ec126a6b85,2019-10-10_23-57-12,1570777032,224.94078087806702,13324.987529277802,21927,ellie,10.0.0.158,13324.987529277802,3240000,54,2,4dce3444,5.587707852952758,1.5389584259441795,0.882305901526306,3234816,3240000,220918.763,10.157,5075.448,4.723,51.45077881619938,38.304361370716514,1.4551915445207286e-12,4.999999873689376e-05,304.02984619140625,7.040964555926621e-05,304.02978515625,-5.7312159462696854e-09,0.0009638385381549597,-0.4713115394115448,0.0 +2788.2348789393354,849.5323429650005,1785.6505683487608,3000.0,20,60000,False,3300000,1100,55,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-00-55,1570777255,223.15795159339905,13548.1454808712,21927,ellie,10.0.0.158,13548.1454808712,3300000,55,2,4dce3444,5.577708271921396,1.5366353519314884,0.8804114250094345,3294720,3300000,220532.096,10.185,5061.38,4.633,51.31792452830188,38.30471698113208,7.275957722603643e-13,4.999999873689376e-05,251.47271728515625,-8.139714918797836e-05,251.4728240966797,-2.0377657339309962e-09,0.0022987022530287504,-0.4628976583480835,0.0 +2788.2348789393354,849.5323429650005,1822.2845389159845,3000.0,20,60000,False,3360000,1120,56,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-04-44,1570777484,228.93655133247375,13777.082032203674,21927,ellie,10.0.0.158,13777.082032203674,3360000,56,2,4dce3444,5.568417337633138,1.5345191609522713,0.8786465458495388,3354624,3360000,220588.654,10.199,5067.143,4.632,51.663914373088694,39.33058103975536,3.6379788613018216e-13,4.999999873689376e-05,293.00921630859375,-0.0003284713311586529,293.0095520019531,-2.0377657339309962e-09,0.005179738160222769,-0.49104171991348267,0.0 +2788.2348789393354,943.0457249174575,1837.8479881899118,3000.0,20,60000,False,3420000,1140,57,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-08-33,1570777713,228.6304793357849,14005.71251153946,21927,ellie,10.0.0.158,14005.71251153946,3420000,57,2,4dce3444,5.559658109902752,1.5326212045498182,0.8769971832419584,3414528,3420000,220734.391,10.192,5064.238,4.609,52.07607361963191,40.06993865030675,1.8189894306509108e-13,4.999999873689376e-05,262.3144226074219,-0.0002233254926977679,262.3146667480469,-8.151062935723985e-09,0.004653227515518665,-0.48141250014305115,0.0 +2788.2348789393354,1013.4801842347422,1851.014425770698,3000.0,20,60000,False,3480000,1160,58,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-12-17,1570777937,224.04734444618225,14229.759855985641,21927,ellie,10.0.0.158,14229.759855985641,3480000,58,2,4dce3444,5.551316000690456,1.5308382366995494,0.8754184791119344,3474432,3480000,220617.594,10.271,5058.143,4.658,51.46551724137931,39.995611285266456,9.094947153254554e-14,4.999999873689376e-05,250.1034393310547,0.00013227085582911968,250.10330200195312,-8.023702591231086e-09,0.0014566911850124598,-0.4811890423297882,0.0 +2649.4182374819,863.929739874468,1804.1985942052709,3000.0,20,60000,False,3540000,1180,59,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-16-08,1570778168,230.6747522354126,14460.434608221054,21927,ellie,10.0.0.158,14460.434608221054,3540000,59,2,4dce3444,5.543606926669899,1.529229142794223,0.8739786483460893,3534336,3540000,220977.904,10.284,5059.191,4.602,54.111854103343475,40.10911854103343,4.547473576627277e-14,4.999999873689376e-05,235.62156677246094,0.000111854009446688,235.62142944335938,3.82081061234274e-10,0.00260219001211226,-0.4805951714515686,0.0 +2635.2257878794317,863.929739874468,1778.3939188339803,3000.0,20,60000,False,3600000,1200,60,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-20-28,1570778428,260.3004310131073,14720.735039234161,21927,ellie,10.0.0.158,14720.735039234161,3600000,60,2,4dce3444,5.538648203625481,1.5285080907098614,0.8731121956970821,3594240,3600000,223877.151,10.874,5401.208,4.615,60.22096774193549,46.51559139784947,2.2737367883136385e-14,4.999999873689376e-05,208.59107971191406,-0.0010167751461267471,208.59210205078125,-4.330252156847791e-09,0.0033729688730090857,-0.5039448142051697,0.0 +2745.0966124556626,863.929739874468,1799.3220629325765,3000.0,20,60000,False,3660000,1220,61,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-26-01,1570778761,333.2357165813446,15053.970755815506,21927,ellie,10.0.0.158,15053.970755815506,3660000,61,2,4dce3444,5.542103984451785,1.5293023327910533,0.8739815761190226,3654144,3660000,234727.62,10.904,5390.841,4.573,69.3861052631579,52.766315789473694,1.1368683941568192e-14,4.999999873689376e-05,319.5943603515625,-0.0012102772016078234,319.5955505371094,5.094414334827491e-10,0.004356545861810446,-0.5843683481216431,0.0 +2745.0966124556626,863.929739874468,1842.7114199770353,3000.0,20,60000,False,3720000,1240,62,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-29-47,1570778987,225.53657627105713,15279.507332086563,21927,ellie,10.0.0.158,15279.507332086563,3720000,62,2,4dce3444,5.545520620895262,1.5299943937683245,0.8748540436520134,3714048,3720000,234838.326,10.896,5389.675,4.496,51.525776397515536,38.588198757763976,5.684341970784096e-15,4.999999873689376e-05,316.5411376953125,-0.00012951325334142894,316.541259765625,-6.495378013227082e-09,0.004084589891135693,-0.5971006751060486,0.0 +2770.5642174531786,863.929739874468,1916.13174319548,3000.0,20,60000,False,3780000,1260,63,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-33-33,1570779213,226.63115096092224,15506.138483047485,21927,ellie,10.0.0.158,15506.138483047485,3780000,63,2,4dce3444,5.54923230028109,1.5307913837053462,0.875794920757431,3773952,3780000,235193.279,10.913,5386.77,4.484,51.44303405572755,38.59380804953561,2.842170985392048e-15,4.999999873689376e-05,349.8982238769531,-0.00020913034677505493,349.8984375,-4.330252156847791e-09,0.002744710072875023,-0.6242475509643555,0.0 +2823.208155771354,950.123074259191,2015.9632386375406,3000.0,20,60000,False,3840000,1280,64,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-37-21,1570779441,227.52810668945312,15733.666589736938,21927,ellie,10.0.0.158,15733.666589736938,3840000,64,2,4dce3444,5.552790642295822,1.5315025825123991,0.8767107927471391,3833856,3840000,235454.431,10.902,5384.433,4.44,51.15432098765432,38.59135802469137,1.421085492696024e-15,4.999999873689376e-05,371.93218994140625,9.162112837657332e-05,371.9320983886719,-5.858576290762585e-09,0.0026667057536542416,-0.6451295018196106,0.0 +2838.3437090532175,1180.449795868128,2140.6203704910176,3000.0,20,60000,False,3900000,1300,65,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-41-09,1570779669,227.74777269363403,15961.414362430573,21927,ellie,10.0.0.158,15961.414362430573,3900000,65,2,4dce3444,5.554262030390675,1.5314606005149167,0.8771664760476363,3893760,3900000,235890.087,10.893,5407.627,4.526,51.29723076923078,38.711384615384624,7.10542746348012e-16,4.999999873689376e-05,394.1192932128906,0.00043883052421733737,394.1188659667969,-2.801927800888393e-09,0.0025074041914194822,-0.6558175086975098,0.0 +2918.3469151650456,1180.449795868128,2161.678808502187,3000.0,20,60000,False,3960000,1320,66,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-44-58,1570779898,229.58401441574097,16190.998376846313,21927,ellie,10.0.0.158,16190.998376846313,3960000,66,2,4dce3444,5.547604467399634,1.5300211382251143,0.8759583229185713,3953664,3960000,235956.784,10.882,5405.598,4.527,51.70823170731707,40.29725609756098,3.55271373174006e-16,4.999999873689376e-05,356.43096923828125,-9.269426664104685e-05,356.43109130859375,-5.858576290762585e-09,0.004095445387065411,-0.6566576957702637,0.0 +2918.3469151650456,1449.8269873641468,2190.5720489630153,3000.0,20,60000,False,4020000,1340,67,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-48-47,1570780127,228.4135136604309,16419.411890506744,21927,ellie,10.0.0.158,16419.411890506744,4020000,67,2,4dce3444,5.5413503492006795,1.52867746695693,0.8748179088503596,4013568,4020000,235936.016,10.937,5404.571,4.591,51.61134969325153,40.223312883435575,1.77635686587003e-16,4.999999873689376e-05,361.1904296875,2.4176051738322712e-05,361.1904296875,-5.858576290762585e-09,0.002447907580062747,-0.6736043095588684,0.0 +2918.3469151650456,1438.7104641756998,2199.050464742624,3000.0,20,60000,False,4080000,1360,68,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-52-35,1570780355,227.9651095867157,16647.37700009346,21927,ellie,10.0.0.158,16647.37700009346,4080000,68,2,4dce3444,5.53538874765602,1.5273949679781345,0.8737303694701037,4073472,4080000,236330.48,10.922,5401.914,4.549,51.61538461538461,40.19753846153845,8.88178432935015e-17,4.999999873689376e-05,359.51202392578125,0.0005820075166411698,359.51141357421875,-3.5660898678457897e-09,0.0026870097499340773,-0.681388795375824,0.0 +2918.3469151650456,1438.7104641756998,2180.1063424663284,3000.0,20,60000,False,4140000,1380,69,fec93e2e40d94966abd856ec126a6b85,2019-10-11_00-56-22,1570780582,227.43166851997375,16874.808668613434,21927,ellie,10.0.0.158,16874.808668613434,4140000,69,2,4dce3444,5.52963380857221,1.5261190507748024,0.8726946942863725,4133376,4140000,236016.01,10.895,5391.981,4.601,51.285802469135795,40.20092592592592,4.440892164675075e-17,4.999999873689376e-05,334.80535888671875,-0.00015563257329631597,334.80548095703125,-2.801927800888393e-09,0.0039174663834273815,-0.6866528391838074,0.0 +2918.3469151650456,1234.8646683984796,2141.4014661967985,3000.0,20,60000,False,4200000,1400,70,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-00-10,1570780810,228.37273383140564,17103.18140244484,21927,ellie,10.0.0.158,17103.18140244484,4200000,70,2,4dce3444,5.524144605373924,1.5249343304998597,0.8717333269251171,4193280,4200000,233176.765,10.321,5039.15,4.554,51.54509202453987,40.19938650306748,2.2204460823375376e-17,4.999999873689376e-05,321.6651611328125,-0.0004485242534428835,321.66558837890625,2.5472071674137453e-10,0.0034829562064260244,-0.6966060996055603,0.0 +2980.2579841057523,1234.8646683984796,2194.1291903129827,3000.0,20,60000,False,4260000,1420,71,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-03-59,1570781039,228.53736877441406,17331.718771219254,21927,ellie,10.0.0.158,17331.718771219254,4260000,71,2,4dce3444,5.518825705227784,1.5236431873458554,0.8707753273153799,4253184,4260000,222703.795,10.323,5042.358,4.515,51.372699386503065,40.19999999999999,1.1102230411687688e-17,4.999999873689376e-05,442.0274963378906,0.00017301113985013217,442.02734375,-3.1840088343670914e-09,0.0018270820146426558,-0.7168861031532288,0.0 +2980.2579841057523,1234.8646683984796,2218.0602102754488,3000.0,20,60000,False,4320000,1440,72,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-07-48,1570781268,228.91155910491943,17560.630330324173,21927,ellie,10.0.0.158,17560.630330324173,4320000,72,2,4dce3444,5.51367882214365,1.5223507510953542,0.8698296614092382,4313088,4320000,223032.386,10.339,5051.127,4.604,51.31009174311927,40.27033639143731,5.551115205843844e-18,4.999999873689376e-05,401.68597412109375,-0.0001589663588674739,401.6861572265625,-2.9292881453812925e-09,0.003532812464982271,-0.7197285294532776,0.0 +2980.2579841057523,1234.8646683984796,2257.353306603134,3000.0,20,60000,False,4380000,1460,73,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-11-37,1570781497,229.3085606098175,17789.93889093399,21927,ellie,10.0.0.158,17789.93889093399,4380000,73,2,4dce3444,5.508814184716648,1.5210246095044329,0.8689136654499581,4372992,4380000,223295.858,10.336,5055.486,4.529,51.407339449541276,40.28990825688074,2.775557602921922e-18,4.999999873689376e-05,424.34722900390625,0.0001054785680025816,424.3470764160156,-3.82081061234274e-10,0.0029879065696150064,-0.7365134358406067,0.0 +2980.2579841057523,1234.8646683984796,2268.0253831030195,3000.0,20,60000,False,4440000,1480,74,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-15-28,1570781728,231.31575798988342,18021.254648923874,21927,ellie,10.0.0.158,18021.254648923874,4440000,74,2,4dce3444,5.504433923819293,1.5197335292932104,0.8680703813076894,4432896,4440000,223670.458,10.328,5059.616,4.528,53.858787878787886,40.31060606060606,1.387778801460961e-18,4.999999873689376e-05,352.5968933105469,2.7753349058912136e-05,352.59686279296875,-7.896342246738186e-09,0.003407558426260948,-0.7281766533851624,0.0 +2980.2579841057523,1324.2961702543648,2302.4423185331275,3000.0,20,60000,False,4500000,1500,75,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-19-17,1570781957,228.7355146408081,18249.990163564682,21927,ellie,10.0.0.158,18249.990163564682,4500000,75,2,4dce3444,5.5002274266484985,1.5184109592396267,0.8672434217662768,4492800,4500000,223780.011,10.328,5049.016,4.453,51.395092024539885,40.45368098159509,6.938894007304805e-19,4.999999873689376e-05,384.5096130371094,0.00026803341461345553,384.50933837890625,-1.7830449339228949e-09,0.002893443452194333,-0.7438187599182129,0.0 +2911.83179233421,1324.2961702543648,2266.767449209455,3000.0,20,60000,False,4560000,1520,76,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-23-03,1570782183,225.30077481269836,18475.29093837738,21927,ellie,10.0.0.158,18475.29093837738,4560000,76,2,4dce3444,5.495962339521445,1.5170809035207744,0.8664305911897956,4552704,4560000,223367.646,10.326,5033.152,4.436,51.52298136645963,40.475776397515524,3.4694470036524025e-19,4.999999873689376e-05,372.38580322265625,0.0002955401432700455,372.385498046875,-2.801927800888393e-09,0.002627121051773429,-0.7563161849975586,0.0 +2884.4544596348105,1324.2961702543648,2236.2977243007317,3000.0,20,60000,False,4620000,1540,77,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-26-50,1570782410,227.0959987640381,18702.38693714142,21927,ellie,10.0.0.158,18702.38693714142,4620000,77,2,4dce3444,5.491742300035577,1.5157512748819035,0.8656393972631606,4612608,4620000,223242.146,10.313,5026.954,4.398,51.49907407407407,40.492901234567896,1.7347235018262012e-19,4.999999873689376e-05,344.80633544921875,-0.0005085475277155638,344.80682373046875,-2.0377657339309962e-09,0.0059786452911794186,-0.7594984173774719,0.0 +3006.42743083957,1324.2961702543648,2177.081601117421,3000.0,20,60000,False,4680000,1560,78,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-30-38,1570782638,227.83968496322632,18930.226622104645,21927,ellie,10.0.0.158,18930.226622104645,4680000,78,2,4dce3444,5.487453627166375,1.5145325896901238,0.8648715494975473,4672512,4680000,223225.799,10.244,5030.901,4.373,51.61015384615384,40.49446153846154,8.673617509131006e-20,4.999999873689376e-05,319.20611572265625,0.0007855209405533969,319.205322265625,2.292486422916795e-09,0.002575932303443551,-0.7642051577568054,0.0 +3006.42743083957,1324.2961702543648,2189.8382442649468,3000.0,20,60000,False,4740000,1580,79,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-34-25,1570782865,227.48921823501587,19157.71584033966,21927,ellie,10.0.0.158,19157.71584033966,4740000,79,2,4dce3444,5.482979743863555,1.5133961910688865,0.8640697710362111,4732416,4740000,223224.944,10.25,5037.512,4.403,51.58364197530864,40.49413580246914,4.336808754565503e-20,4.999999873689376e-05,372.6768798828125,-0.00024281104560941458,372.6771545410156,-7.641621557752387e-09,0.003701510140672326,-0.7932894229888916,0.0 +3006.42743083957,1452.764198488252,2157.248776174541,3000.0,20,60000,False,4800000,1600,80,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-38-11,1570783091,226.3192265033722,19384.035066843033,21927,ellie,10.0.0.158,19384.035066843033,4800000,80,2,4dce3444,5.4784927745283305,1.5122587457884216,0.863272640111636,4792320,4800000,223018.747,10.185,5038.609,4.295,51.45665634674923,40.49752321981424,2.1684043772827515e-20,4.999999873689376e-05,326.7668151855469,-1.17258769023465e-05,326.766845703125,-1.9104053894380968e-09,0.004748365841805935,-0.7888689041137695,0.0 +3006.42743083957,1332.1297631757272,2183.5909989627335,3000.0,20,60000,False,4860000,1620,81,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-42-01,1570783321,230.04490852355957,19614.079975366592,21927,ellie,10.0.0.158,19614.079975366592,4860000,81,2,4dce3444,5.474336296065456,1.5113205798127114,0.8625214706630987,4852224,4860000,223159.356,10.118,5048.353,4.8,51.63384146341464,40.49603658536586,1.0842021886413758e-20,4.999999873689376e-05,413.1332702636719,-0.0002729884581640363,413.1335754394531,-6.622738357719982e-09,0.0043872338719666,-0.8144399523735046,0.0 +3006.42743083957,1332.1297631757272,2234.8340329616385,3000.0,20,60000,False,4920000,1640,82,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-45-51,1570783551,229.55120015144348,19843.631175518036,21927,ellie,10.0.0.158,19843.631175518036,4920000,82,2,4dce3444,5.470398736536834,1.5104265040217797,0.8617891469656084,4912128,4920000,223228.435,10.089,5043.321,4.774,51.297256097560975,40.5,5.421010943206879e-21,4.999999873689376e-05,426.8544921875,-0.00012361582776065916,426.8546447753906,-5.094414223805188e-09,0.0035946981515735388,-0.824469268321991,0.0 +2933.967885678605,1332.1297631757272,2219.425534777116,3000.0,20,60000,False,4980000,1660,83,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-49-36,1570783776,225.3933732509613,20069.024548768997,21927,ellie,10.0.0.158,20069.024548768997,4980000,83,2,4dce3444,5.4664105812545,1.5094611079483513,0.8610582265066498,4972032,4980000,222835.053,10.041,5045.128,4.84,51.38411214953271,40.498753894081,2.7105054716034394e-21,4.999999873689376e-05,298.0599060058594,-0.00013286175089888275,298.0600280761719,-4.202891812354892e-09,0.0036338898353278637,-0.7899288535118103,0.0 +2916.691916413175,1332.1297631757272,2202.864556787631,3000.0,20,60000,False,5040000,1680,84,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-53-24,1570784004,227.5010061264038,20296.5255548954,21927,ellie,10.0.0.158,20296.5255548954,5040000,84,2,4dce3444,5.462527868617465,1.508453193103734,0.8603528209390059,5031936,5040000,222449.213,10.05,5049.473,4.829,51.315384615384616,40.49846153846154,1.3552527358017197e-21,4.999999873689376e-05,334.1376953125,-0.00046916879364289343,334.1381530761719,-4.457612501340691e-09,0.00451930146664381,-0.819091796875,0.0 +2905.747602907159,1332.1297631757272,2179.710890083026,3000.0,20,60000,False,5100000,1700,85,fec93e2e40d94966abd856ec126a6b85,2019-10-11_01-57-11,1570784231,227.45634031295776,20523.98189520836,21927,ellie,10.0.0.158,20523.98189520836,5100000,85,2,4dce3444,5.4587606544091845,1.5075156912487153,0.8596746607010859,5091840,5100000,222331.737,10.059,5039.01,4.823,51.41851851851851,40.498765432098764,6.776263679008599e-22,4.999999873689376e-05,283.073974609375,0.0002259069588035345,283.07373046875,-1.1080350859060673e-08,0.0033177239820361137,-0.8060696721076965,0.0 +2905.747602907159,1196.2430041476587,2104.488545126206,3000.0,20,60000,False,5160000,1720,86,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-00-57,1570784457,225.21863865852356,20749.200533866882,21927,ellie,10.0.0.158,20749.200533866882,5160000,86,2,4dce3444,5.454831173362524,1.5064578514085258,0.8589962138263258,5151744,5160000,222324.584,10.02,5038.046,4.813,51.46323987538941,40.49906542056075,3.3881318395042993e-22,4.999999873689376e-05,291.6480407714844,-4.706958497990854e-05,291.6481018066406,-3.056648489874192e-09,0.003895676461979747,-0.8070542812347412,0.0 +2899.148447687858,1196.2430041476587,2052.3106909380394,3000.0,20,60000,False,5220000,1740,87,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-04-44,1570784684,227.65292978286743,20976.85346364975,21927,ellie,10.0.0.158,20976.85346364975,5220000,87,2,4dce3444,5.450893973983236,1.5054460582614069,0.8583462908434553,5211648,5220000,222378.112,9.955,5040.267,4.804,51.559384615384616,40.5,1.6940659197521496e-22,4.999999873689376e-05,336.4368896484375,0.00021401583217084408,336.43670654296875,-5.985936635255484e-09,0.002751996275037527,-0.8360207080841064,0.0 +2758.315550076924,1013.5978424744745,2035.4545333821252,3000.0,20,60000,False,5280000,1760,88,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-08-31,1570784911,227.0863552093506,21203.9398188591,21927,ellie,10.0.0.158,21203.9398188591,5280000,88,2,4dce3444,5.447115531919371,1.5045399413494056,0.8577291555253139,5271552,5280000,222299.236,10.005,5043.785,4.751,51.49660493827161,40.5,8.470329598760748e-23,4.999999873689376e-05,271.9049072265625,0.0003052720567211509,271.90460205078125,-1.2226593959496768e-08,0.003503115614876151,-0.8166009187698364,0.0 +2737.0613425937045,1013.5978424744745,2026.995016320426,3000.0,20,60000,False,5340000,1780,89,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-12-19,1570785139,227.37750267982483,21431.317321538925,21927,ellie,10.0.0.158,21431.317321538925,5340000,89,2,4dce3444,5.443448861807843,1.503688345148238,0.8571220444556282,5331456,5340000,222291.746,9.997,5040.162,4.672,51.66461538461539,40.5,4.235164799380374e-23,4.999999873689376e-05,331.7037353515625,-0.000127263818285428,331.703857421875,-4.0755314678619925e-09,0.004985067993402481,-0.8520377278327942,0.0 +2737.0613425937045,964.4086085056584,1999.0844661466292,3000.0,20,60000,False,5400000,1800,90,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-17-27,1570785447,308.3288314342499,21739.646152973175,21927,ellie,10.0.0.158,21739.646152973175,5400000,90,2,4dce3444,5.443818108807803,1.5037491850696432,0.8573441394371691,5391360,5400000,230063.943,10.431,5466.461,4.782,67.48268792710707,48.08815489749431,2.117582399690187e-23,4.999999873689376e-05,252.10047912597656,-0.00036170537350699306,252.10084533691406,-2.6745674563954935e-09,0.0037506765220314264,-0.8334175944328308,0.0 +2737.0613425937045,964.4086085056584,1962.2322116681294,3000.0,20,60000,False,5460000,1820,91,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-22-54,1570785774,327.270468711853,22066.916621685028,21927,ellie,10.0.0.158,22066.916621685028,5460000,91,2,4dce3444,5.449222481375564,1.504937074639416,0.8585842609888559,5451264,5460000,239495.747,10.873,5756.357,4.534,69.03276231263382,52.65867237687366,1.0587911998450935e-23,4.999999873689376e-05,230.7174072265625,0.0004092047456651926,230.71697998046875,-7.641621557752387e-09,0.0024183830246329308,-0.8498794436454773,0.0 +2677.557837368173,964.4086085056584,1940.4812706349296,3000.0,20,60000,False,5520000,1840,92,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-28-01,1570786081,306.19391202926636,22373.110533714294,21927,ellie,10.0.0.158,22373.110533714294,5520000,92,2,4dce3444,5.458546806651846,1.507038553790822,0.8606399314970993,5511168,5520000,247162.509,10.884,5753.814,4.579,65.68878718535468,52.97528604118993,5.293955999225468e-24,4.999999873689376e-05,297.43548583984375,-0.00034542870707809925,297.435791015625,-4.71233319032649e-09,0.004611298441886902,-0.8941606283187866,0.0 +2683.5308525802825,964.4086085056584,1957.650974117936,3000.0,20,60000,False,5580000,1860,93,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-31-46,1570786306,225.56541466712952,22598.675948381424,21927,ellie,10.0.0.158,22598.675948381424,5580000,93,2,4dce3444,5.467778906102963,1.5091149986232149,0.8626515789327578,5571072,5580000,247183.82,10.993,5749.685,4.529,51.570093457943926,46.34517133956386,2.646977999612734e-24,4.999999873689376e-05,290.9716796875,-0.0004666796594392508,290.97216796875,-6.750098702212881e-09,0.0057234009727835655,-0.8967065811157227,0.0 +2683.5308525802825,964.4086085056584,1939.0772353866698,3000.0,20,60000,False,5640000,1880,94,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-35-31,1570786531,224.76352715492249,22823.439475536346,21927,ellie,10.0.0.158,22823.439475536346,5640000,94,2,4dce3444,5.47675565317096,1.5111543505451976,0.8646343724194111,5630976,5640000,246921.379,11.012,5738.348,4.572,51.34610591900311,46.16386292834892,1.323488999806367e-24,4.999999873689376e-05,290.494873046875,0.00038343845517374575,290.4944763183594,-7.514261213259488e-09,0.004202503245323896,-0.9158763289451599,0.0 +2861.060956793371,1256.0643935563871,2000.0722688135281,3000.0,20,60000,False,5700000,1900,95,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-39-16,1570786756,225.32696080207825,23048.766436338425,21927,ellie,10.0.0.158,23048.766436338425,5700000,95,2,4dce3444,5.481652038325564,1.5122527184887817,0.8657620794464368,5690880,5700000,246709.273,10.982,5737.554,4.54,51.388161993769465,46.148286604361374,6.617444999031835e-25,4.999999873689376e-05,329.7452392578125,3.619683047872968e-05,329.74517822265625,-2.292486422916795e-09,0.005399812012910843,-0.9454441070556641,0.0 +2861.060956793371,1290.5253107259348,2035.020953785304,3000.0,20,60000,False,5760000,1920,96,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-43-04,1570786984,227.3152379989624,23276.081674337387,21927,ellie,10.0.0.158,23276.081674337387,5760000,96,2,4dce3444,5.481518327121863,1.5123113546300606,0.8658710432810979,5750784,5760000,246878.635,11.056,5777.405,4.555,51.31784615384616,46.52492307692307,3.3087224995159173e-25,4.999999873689376e-05,286.93316650390625,-0.0002915517834480852,286.9334411621094,-2.9292881453812925e-09,0.0048973518423736095,-0.9249772429466248,0.0 +2934.826069042796,1290.5253107259348,2063.7627740994685,3000.0,20,60000,False,5820000,1940,97,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-46-52,1570787212,228.78414392471313,23504.8658182621,21927,ellie,10.0.0.158,23504.8658182621,5820000,97,2,4dce3444,5.477459937742354,1.5114917081981534,0.8651604433805758,5810688,5820000,246975.292,11.151,5793.693,4.622,51.444478527607366,47.99110429447853,1.6543612497579586e-25,4.999999873689376e-05,345.74163818359375,0.00039254192961379886,345.7412414550781,-7.386900424677378e-09,0.00418316712602973,-0.9660893678665161,0.0 +2934.826069042796,1290.5253107259348,2093.4478175053637,3000.0,20,60000,False,5880000,1960,98,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-50-40,1570787440,227.26147627830505,23732.127294540405,21927,ellie,10.0.0.158,23732.127294540405,5880000,98,2,4dce3444,5.473579763514985,1.510672495994402,0.8644802404398336,5870592,5880000,246999.299,11.149,5787.114,4.698,51.58240740740741,47.92777777777777,8.271806248789793e-26,4.999999873689376e-05,329.9728698730469,-0.0009427159675396979,329.9737854003906,-4.839693534819389e-09,0.005523227620869875,-0.9740684032440186,0.0 +2934.826069042796,1170.9146099550221,2079.6405859727024,3000.0,20,60000,False,5940000,1980,99,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-54-26,1570787666,226.66594099998474,23958.79323554039,21927,ellie,10.0.0.158,23958.79323554039,5940000,99,2,4dce3444,5.469870841172653,1.5099312980036705,0.8638283152478821,5930496,5940000,246926.178,11.115,5789.075,4.75,51.598765432098766,47.976851851851855,4.1359031243948966e-26,4.999999873689376e-05,276.3343200683594,0.00035375356674194336,276.3339538574219,-3.1840088343670914e-09,0.003470742842182517,-0.9474529027938843,0.0 +2934.826069042796,1170.9146099550221,2059.6971649764732,3000.0,20,60000,False,6000000,2000,100,fec93e2e40d94966abd856ec126a6b85,2019-10-11_02-58-14,1570787894,227.67542624473572,24186.468661785126,21927,ellie,10.0.0.158,24186.468661785126,6000000,100,2,4dce3444,5.466263041875936,1.509240417837348,0.8632060163710099,5990400,6000000,239290.315,10.675,5361.964,4.705,51.16635802469136,47.998765432098764,2.0679515621974483e-26,4.999999873689376e-05,297.5181884765625,-0.00040703132981434464,297.51861572265625,5.094414334827491e-10,0.005609925836324692,-0.9748757481575012,0.0 +2934.826069042796,1170.9146099550221,2040.987912493069,3000.0,20,60000,False,6060000,2020,101,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-01-59,1570788119,225.31001806259155,24411.778679847717,21927,ellie,10.0.0.158,24411.778679847717,6060000,101,2,4dce3444,5.462698785768646,1.5085268600760924,0.8626049430374765,6050304,6060000,229393.23,10.251,5064.066,4.635,51.42173913043478,48.000621118012425,1.0339757810987241e-26,4.999999873689376e-05,254.15103149414062,0.00040054015698842704,254.15060424804688,-1.273603555951297e-09,0.0036605566274374723,-0.9686458706855774,0.0 +2809.436008132594,1140.723034175603,1995.5661046800012,3000.0,20,60000,False,6120000,2040,102,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-05-45,1570788345,225.23614883422852,24637.014828681946,21927,ellie,10.0.0.158,24637.014828681946,6120000,102,2,4dce3444,5.459090960330461,1.5077626930834152,0.8620029306174886,6110208,6120000,221298.993,10.239,5062.694,4.501,51.4373831775701,48.06292834890966,5.169878905493621e-27,4.999999873689376e-05,274.35589599609375,2.6937734219245613e-05,274.3558654785156,-5.985936635255484e-09,0.005472501739859581,-0.9897372722625732,0.0 +2790.561620110469,1140.723034175603,1941.9584758988715,3000.0,20,60000,False,6180000,2060,103,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-09-31,1570788571,226.39943838119507,24863.41426706314,21927,ellie,10.0.0.158,24863.41426706314,6180000,103,2,4dce3444,5.455476650917437,1.5070224047497909,0.8614178635704232,6170112,6180000,221370.141,10.144,5075.015,4.547,51.57337461300311,48.09752321981425,2.5849394527468104e-27,4.999999873689376e-05,251.54049682617188,9.814197255764157e-06,251.54049682617188,-1.0443549136596175e-08,0.003990766126662493,-0.9907731413841248,0.0 +2730.1414912893997,1065.1849980808593,1917.6636954166001,3000.0,20,60000,False,6240000,2080,104,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-13-16,1570788796,224.90760898590088,25088.32187604904,21927,ellie,10.0.0.158,25088.32187604904,6240000,104,2,4dce3444,5.451870524366552,1.506245161603624,0.860839929966134,6230016,6240000,221377.192,10.143,5082.406,4.521,51.642679127725856,48.10155763239876,1.2924697263734052e-27,4.999999873689376e-05,244.52688598632812,0.00037902427720837295,244.52650451660156,-7.386900424677378e-09,0.0040008919313549995,-0.9825717806816101,0.0 +2656.880583875099,1065.1849980808593,1923.6063345284313,3000.0,20,60000,False,6300000,2100,105,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-17-08,1570789028,231.9039261341095,25320.22580218315,21927,ellie,10.0.0.158,25320.22580218315,6300000,105,2,4dce3444,5.4485876723697535,1.5055536435757335,0.8603106469197712,6289920,6300000,222031.126,10.243,5086.015,4.577,54.30906344410877,48.616616314199405,6.462348631867026e-28,4.999999873689376e-05,305.9954528808594,0.00037878184230066836,305.9950866699219,-1.0188828447610376e-08,0.0034544565714895725,-1.0335384607315063,0.0 +2656.880583875099,1065.1849980808593,1931.7140297846706,3000.0,20,60000,False,6360000,2120,106,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-20-56,1570789256,227.864901304245,25548.090703487396,21927,ellie,10.0.0.158,25548.090703487396,6360000,106,2,4dce3444,5.445487747192605,1.5048747807639749,0.8598048336385432,6349824,6360000,222122.505,10.153,5050.036,4.573,51.757538461538466,48.802153846153836,3.231174315933513e-28,4.999999873689376e-05,268.95941162109375,0.0002717429306358099,268.95916748046875,-9.29730603616008e-09,0.0040903775952756405,-1.017516851425171,0.0 +2656.880583875099,1065.1849980808593,1944.8068969692508,3000.0,20,60000,False,6420000,2140,107,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-24-43,1570789483,227.35837364196777,25775.449077129364,21927,ellie,10.0.0.158,25775.449077129364,6420000,107,2,4dce3444,5.442537109313096,1.5042352542568365,0.8593207328452318,6409728,6420000,221992.431,10.146,5037.708,4.367,51.40586419753087,48.73240740740741,1.6155871579667565e-28,4.999999873689376e-05,291.67462158203125,-0.00012354386853985488,291.6747131347656,-4.839693534819389e-09,0.004860265646129847,-1.0397971868515015,0.0 +2712.1523702462287,1065.1849980808593,1943.004044728714,3000.0,20,60000,False,6480000,2160,108,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-28-28,1570789708,224.78979206085205,26000.238869190216,21927,ellie,10.0.0.158,26000.238869190216,6480000,108,2,4dce3444,5.439569044549453,1.5035642722912863,0.8588395121510299,6469632,6480000,221746.881,10.142,5036.103,4.383,51.34735202492212,48.79283489096574,8.077935789833782e-29,4.999999873689376e-05,249.88047790527344,0.0002704112557694316,249.8802032470703,-5.349134912790987e-09,0.0048729111440479755,-1.0350946187973022,0.0 +2712.1523702462287,1223.9570474460438,1953.0073051284094,3000.0,20,60000,False,6540000,2180,109,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-32-12,1570789932,224.17111325263977,26224.409982442856,21927,ellie,10.0.0.158,26224.409982442856,6540000,109,2,4dce3444,5.436593404466541,1.502886891182002,0.8583622739880797,6529536,6540000,221498.282,10.133,5035.313,4.325,51.284687500000004,48.80062500000001,4.038967894916891e-29,4.999999873689376e-05,245.7689666748047,0.00016378675354644656,245.76878356933594,-2.1651260784238957e-09,0.005801717285066843,-1.0496704578399658,0.0 +2772.7075470226778,1223.9570474460438,1954.9965068179956,3000.0,20,60000,False,6600000,2200,110,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-35-59,1570790159,226.57998132705688,26450.989963769913,21927,ellie,10.0.0.158,26450.989963769913,6600000,110,2,4dce3444,5.433396673676116,1.5021347924239057,0.8578479544411897,6589440,6600000,221391.037,10.122,5032.943,4.381,51.28018575851393,48.86130030959752,2.0194839474584456e-29,4.999999873689376e-05,307.3695983886719,0.0003561436606105417,307.3692626953125,-2.6745674563954935e-09,0.0036990707740187645,-1.082297682762146,0.0 +2884.8556156266472,1223.9570474460438,2004.2035484929286,3000.0,20,60000,False,6660000,2220,111,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-39-47,1570790387,228.4696180820465,26679.45958185196,21927,ellie,10.0.0.158,26679.45958185196,6660000,111,2,4dce3444,5.430316922296806,1.5014319947783252,0.8573416353333044,6649344,6660000,221709.603,10.144,5030.409,4.287,51.52791411042944,48.896932515337404,1.0097419737292228e-29,4.999999873689376e-05,332.7329406738281,0.00028941722121089697,332.7326965332031,-5.603855601776786e-09,0.005987292621284723,-1.1064846515655518,0.0 +2940.7799109831026,1308.0179210304186,2055.9638943096447,3000.0,20,60000,False,6720000,2240,112,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-43-35,1570790615,227.94732356071472,26907.406905412674,21927,ellie,10.0.0.158,26907.406905412674,6720000,112,2,4dce3444,5.427364009165221,1.5007114698388222,0.8568438725398819,6709248,6720000,221980.667,10.181,5029.406,5.276,51.566769230769225,48.90523076923076,5.048709868646114e-30,4.999999873689376e-05,369.8870849609375,0.0002594511315692216,369.8868103027344,-1.1335071548046471e-08,0.004924853798002005,-1.1358833312988281,0.0 +2940.7799109831026,1343.7128477823323,2102.4146581586006,3000.0,20,60000,False,6780000,2260,113,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-47-22,1570790842,227.20156455039978,27134.608469963074,21927,ellie,10.0.0.158,27134.608469963074,6780000,113,2,4dce3444,5.424558813034257,1.500031745123004,0.8563517802194234,6769152,6780000,222069.452,10.224,5020.797,5.251,51.27746913580248,48.99413580246914,2.524354934323057e-30,4.999999873689376e-05,316.38421630859375,0.00031902099726721644,316.3838806152344,-1.7830449339228949e-09,0.0041290284134447575,-1.1200151443481445,0.0 +2940.7799109831026,1121.422021120273,2123.797024652828,3000.0,20,60000,False,6840000,2280,114,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-51-09,1570791069,226.85546731948853,27361.463937282562,21927,ellie,10.0.0.158,27361.463937282562,6840000,114,2,4dce3444,5.42194423287052,1.499433433427161,0.8558800953311189,6829056,6840000,222263.381,10.219,5020.681,5.247,51.62376543209877,48.998765432098764,1.2621774671615285e-30,4.999999873689376e-05,280.3107604980469,0.0005952796782366931,280.3101501464844,-3.8208107788761936e-09,0.004510277882218361,-1.1148210763931274,0.0 +2940.7799109831026,1121.422021120273,2142.480614877029,3000.0,20,60000,False,6900000,2300,115,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-54-58,1570791298,228.81058764457703,27590.27452492714,21927,ellie,10.0.0.158,27590.27452492714,6900000,115,2,4dce3444,5.4195130422937146,1.4988723221142186,0.8554391231282957,6888960,6900000,221949.306,10.093,5025.647,5.143,51.58006134969326,49.04141104294479,6.3108873358076425e-31,4.999999873689376e-05,334.6033020019531,-8.271870319731534e-05,334.6034240722656,6.368017779756485e-10,0.005179248284548521,-1.153441309928894,0.0 +2940.7799109831026,1121.422021120273,2119.688029785074,3000.0,20,60000,False,6960000,2320,116,fec93e2e40d94966abd856ec126a6b85,2019-10-11_03-58-45,1570791525,227.1993865966797,27817.47391152382,21927,ellie,10.0.0.158,27817.47391152382,6960000,116,2,4dce3444,5.41706190677346,1.498304746547766,0.8550045682509223,6948864,6960000,221885.029,10.276,5023.075,5.045,51.53487654320988,49.10432098765433,3.1554436679038213e-31,4.999999873689376e-05,299.1117858886719,0.00014942610869184136,299.111572265625,-7.641621557752387e-09,0.0030115745030343533,-1.1431351900100708,0.0 +2941.833458681214,1121.422021120273,2090.0029044763705,3000.0,20,60000,False,7020000,2340,117,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-02-32,1570791752,227.13538074493408,28044.609292268753,21927,ellie,10.0.0.158,28044.609292268753,7020000,117,2,4dce3444,5.414588395015474,1.497791611688474,0.8545688636184476,7008768,7020000,221866.764,10.268,5019.012,5.107,51.54012345679013,49.19197530864197,1.5777218339519106e-31,4.999999873689376e-05,325.9025573730469,-0.0001517296041129157,325.9027099609375,-8.151062935723985e-09,0.005776040721684694,-1.1616567373275757,0.0 +2941.833458681214,1121.422021120273,2093.3040627928413,3000.0,20,60000,False,7080000,2360,118,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-06-19,1570791979,227.06834149360657,28271.67763376236,21927,ellie,10.0.0.158,28271.67763376236,7080000,118,2,4dce3444,5.412179671864106,1.497276988574628,0.8541445187758996,7068672,7080000,222092.172,10.369,5021.272,5.154,51.39012345679013,49.09382716049383,7.888609169759553e-32,4.999999873689376e-05,318.9958190917969,0.000534237886313349,318.99530029296875,-4.202891812354892e-09,0.004300462082028389,-1.1741962432861328,0.0 +2941.833458681214,1381.8246345037035,2132.742663850571,3000.0,20,60000,False,7140000,2380,119,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-10-06,1570792206,226.6509449481964,28498.328578710556,21927,ellie,10.0.0.158,28498.328578710556,7140000,119,2,4dce3444,5.409818033954657,1.496748968202069,0.8537245284535437,7128576,7140000,222342.109,10.371,5019.324,5.17,51.58858024691358,49.12746913580248,3.9443045848797766e-32,4.999999873689376e-05,328.4617614746094,0.00026661311858333647,328.46148681640625,-7.00481939119868e-09,0.00501100393012166,-1.196395993232727,0.0 +3007.2657019383378,1381.8246345037035,2142.2210019576232,3000.0,20,60000,False,7200000,2400,120,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-13-55,1570792435,228.56880950927734,28726.897388219833,21927,ellie,10.0.0.158,28726.897388219833,7200000,120,2,4dce3444,5.407467040365899,1.4962414689426806,0.8533092066350583,7188480,7200000,222532.735,10.424,5027.548,5.139,51.700306748466254,49.20184049079754,1.9721522924398883e-32,4.999999873689376e-05,350.4010314941406,0.0007466197712346911,350.4002990722656,-8.405783624709784e-09,0.003580022370442748,-1.2104030847549438,0.0 +3035.9625665117865,1381.8246345037035,2188.7578497428026,3000.0,20,60000,False,7260000,2420,121,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-17-46,1570792666,231.5637137889862,28958.46110200882,21927,ellie,10.0.0.158,28958.46110200882,7260000,121,2,4dce3444,5.405286999276755,1.4957948552218552,0.8529035167047238,7248384,7260000,222843.99,10.438,5025.774,5.092,53.22878787878788,49.452121212121206,9.860761462199441e-33,4.999999873689376e-05,373.27130126953125,0.0007099591894075274,373.27056884765625,-9.042585347174281e-09,0.004816744010895491,-1.2301075458526611,0.0 +3057.3607231299898,1524.9067550215464,2208.2048086221653,3000.0,20,60000,False,7320000,2440,122,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-21-36,1570792896,229.34732294082642,29187.808424949646,21927,ellie,10.0.0.158,29187.808424949646,7320000,122,2,4dce3444,5.40327161899733,1.4953220983408837,0.8525253518729536,7308288,7320000,222980.438,10.443,5030.24,4.158,52.535168195718654,51.034862385321105,4.930380731099721e-33,4.999999873689376e-05,349.9038391113281,0.0003130502300336957,349.90350341796875,-6.495378013227082e-09,0.005526693072170019,-1.2306982278823853,0.0 +3057.3607231299898,1413.2260015039808,2212.461681829032,3000.0,20,60000,False,7380000,2460,123,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-25-21,1570793121,225.77511978149414,29413.58354473114,21927,ellie,10.0.0.158,29413.58354473114,7380000,123,2,4dce3444,5.401256955778731,1.494855449356993,0.8521448129227898,7368192,7380000,222836.759,10.394,5031.256,4.224,51.517080745341616,49.963043478260865,2.4651903655498604e-33,4.999999873689376e-05,324.2953796386719,-0.0001201698396471329,324.29547119140625,-3.056648489874192e-09,0.005093560088425875,-1.2325332164764404,0.0 +3057.3607231299898,1321.103810498718,2207.9320675315703,3000.0,20,60000,False,7440000,2480,124,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-29-08,1570793348,227.08454179763794,29640.668086528778,21927,ellie,10.0.0.158,29640.668086528778,7440000,124,2,4dce3444,5.399314561093524,1.4943760066727925,0.8517784762022178,7428096,7440000,222863.773,10.388,5028.116,4.212,51.53456790123457,49.998456790123456,1.2325951827749302e-33,4.999999873689376e-05,327.2377014160156,0.0007862969650886953,327.2369079589844,-7.896342246738186e-09,0.004149282816797495,-1.2451683282852173,0.0 +3057.3607231299898,1321.103810498718,2211.0565058699813,3000.0,20,60000,False,7500000,2500,125,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-32-55,1570793575,226.42987155914307,29867.09795808792,21927,ellie,10.0.0.158,29867.09795808792,7500000,125,2,4dce3444,5.397341603606959,1.493854482234554,0.851396021710565,7488000,7500000,222627.347,10.461,5026.169,4.391,51.38452012383901,50.013931888544896,6.162975913874651e-34,4.999999873689376e-05,349.337158203125,0.0010858766036108136,349.3360595703125,1.0188828669654981e-09,0.004914838820695877,-1.2671468257904053,0.0 +3057.3607231299898,1321.103810498718,2183.928197300355,3000.0,20,60000,False,7560000,2520,126,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-36-40,1570793800,225.5946009159088,30092.69255900383,21927,ellie,10.0.0.158,30092.69255900383,7560000,126,2,4dce3444,5.395215937311672,1.4932358484074018,0.8509987763666014,7547904,7560000,222465.163,10.33,5028.049,4.52,51.29316770186336,50.098757763975165,3.0814879569373254e-34,4.999999873689376e-05,322.60345458984375,8.859772060532123e-05,322.6033630371094,-1.273603555951297e-09,0.005859797820448875,-1.270148515701294,0.0 +3033.4231468098355,1321.103810498718,2179.9617275406704,3000.0,20,60000,False,7620000,2540,127,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-40-28,1570794028,227.34414887428284,30320.036707878113,21927,ellie,10.0.0.158,30320.036707878113,7620000,127,2,4dce3444,5.392951297448699,1.4926434263958868,0.8505888567590937,7607808,7620000,222488.67,10.277,5025.364,4.657,51.18055555555556,50.14382716049382,1.5407439784686627e-34,4.999999873689376e-05,338.9652099609375,0.0004025684902444482,338.9648132324219,-4.839693534819389e-09,0.005759512074291706,-1.2807590961456299,0.0 +3037.4853168816417,1321.103810498718,2198.406143377301,3000.0,20,60000,False,7680000,2560,128,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-44-16,1570794256,228.3664309978485,30548.40313887596,21927,ellie,10.0.0.158,30548.40313887596,7680000,128,2,4dce3444,5.390797719972908,1.4920736276528817,0.8501934479448069,7667712,7680000,222617.042,10.156,5026.938,4.622,51.4021472392638,50.18742331288343,7.703719892343314e-35,4.999999873689376e-05,352.56549072265625,0.00034955545561388135,352.56512451171875,-1.0952990514567773e-08,0.004894816316664219,-1.2931183576583862,0.0 +3037.4853168816417,1476.5492618048222,2242.1038809597444,3000.0,20,60000,False,7740000,2580,129,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-48-06,1570794486,229.57926726341248,30777.982406139374,21927,ellie,10.0.0.158,30777.982406139374,7740000,129,2,4dce3444,5.388764883258537,1.4915607167820297,0.8498014428754362,7727616,7740000,222912.816,10.193,5023.862,4.646,51.64512195121951,50.200914634146336,3.851859946171657e-35,4.999999873689376e-05,394.87640380859375,2.344856875424739e-05,394.8763732910156,-3.693450212338689e-09,0.0066440817900002,-1.331769347190857,0.0 +3037.4853168816417,1476.5492618048222,2230.8651329685163,3000.0,20,60000,False,7800000,2600,130,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-51-53,1570794713,226.72895216941833,31004.711358308792,21927,ellie,10.0.0.158,31004.711358308792,7800000,130,2,4dce3444,5.38676642905989,1.4910501694570497,0.8494238213301806,7787520,7800000,222731.956,10.214,5020.702,4.659,51.349535603715175,50.1938080495356,1.9259299730858284e-35,4.999999873689376e-05,331.77386474609375,0.00028852379182353616,331.7735595703125,-8.660504313695583e-09,0.0059051718562841415,-1.3130464553833008,0.0 +3037.4853168816417,1476.5492618048222,2238.2221261098075,3000.0,20,60000,False,7860000,2620,131,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-55-40,1570794940,227.29516005516052,31232.006518363953,21927,ellie,10.0.0.158,31232.006518363953,7860000,131,2,4dce3444,5.384849433361234,1.4905851723290888,0.8490553262881387,7847424,7860000,222300.144,10.157,5025.665,4.711,51.313580246913574,50.19814814814814,9.629649865429142e-36,4.999999873689376e-05,333.77398681640625,2.017731821979396e-05,333.77398681640625,-4.0755314678619925e-09,0.00595882348716259,-1.324370265007019,0.0 +3037.4853168816417,1315.4813396690183,2198.8328680971335,3000.0,20,60000,False,7920000,2640,132,fec93e2e40d94966abd856ec126a6b85,2019-10-11_04-59-25,1570795165,224.9325294494629,31456.939047813416,21927,ellie,10.0.0.158,31456.939047813416,7920000,132,2,4dce3444,5.382926391695807,1.4900840726120073,0.848692482690214,7907328,7920000,221856.045,10.101,5028.389,4.699,51.35794392523365,50.19968847352025,4.814824932714571e-36,4.999999873689376e-05,284.7213439941406,0.0001491272560087964,284.7212219238281,-4.202891812354892e-09,0.005539558827877045,-1.2887325286865234,0.0 +2914.4994178660495,1315.4813396690183,2210.8419579488987,3000.0,20,60000,False,7980000,2660,133,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-03-13,1570795393,228.15981030464172,31685.098858118057,21927,ellie,10.0.0.158,31685.098858118057,7980000,133,2,4dce3444,5.381025311447718,1.4895990205230913,0.8483285461262264,7967232,7980000,222093.16,10.152,5029.651,4.685,51.30490797546012,50.20644171779141,2.4074124663572855e-36,4.999999873689376e-05,374.6876525878906,-0.00011911516776308417,374.68780517578125,-1.1462431892539371e-08,0.006542982999235392,-1.3549070358276367,0.0 +2914.4994178660495,1315.4813396690183,2147.2172593307414,3000.0,20,60000,False,8040000,2680,134,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-06-59,1570795619,226.0365846157074,31911.135442733765,21927,ellie,10.0.0.158,31911.135442733765,8040000,134,2,4dce3444,5.379007892396017,1.4890622643743392,0.847965638016519,8027136,8040000,221986.763,10.165,5031.25,4.716,51.468322981366455,50.292236024844726,1.2037062331786428e-36,4.999999873689376e-05,299.0262145996094,0.0002473544445820153,299.02593994140625,-7.896342246738186e-09,0.004999789409339428,-1.3221986293792725,0.0 +2914.4994178660495,1299.1913996940846,2097.3217711064494,3000.0,20,60000,False,8100000,2700,135,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-10-45,1570795845,226.36364698410034,32137.499089717865,21927,ellie,10.0.0.158,32137.499089717865,8100000,135,2,4dce3444,5.376944275509544,1.4885588587970249,0.8476114081224412,8087040,8100000,221983.904,10.156,5027.588,4.67,51.24705882352942,50.29938080495357,6.018531165893214e-37,4.999999873689376e-05,266.26568603515625,0.00022065282973926514,266.2654724121094,-4.967053879312289e-09,0.004009648226201534,-1.2973803281784058,0.0 +3004.663396526677,1299.1913996940846,2125.072402909051,3000.0,20,60000,False,8160000,2720,136,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-14-36,1570796076,230.6580309867859,32368.15712070465,21927,ellie,10.0.0.158,32368.15712070465,8160000,136,2,4dce3444,5.3749733216627895,1.488075023046842,0.8472679770651395,8146944,8160000,222287.343,10.275,5229.981,4.604,52.170212765957444,50.35410334346505,3.009265582946607e-37,4.999999873689376e-05,378.9477844238281,0.0001447344693588093,378.9476623535156,-5.603855601776786e-09,0.006529606878757477,-1.3991847038269043,0.0 +3004.663396526677,1244.613686736468,2163.625640905617,3000.0,20,60000,False,8220000,2740,137,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-18-29,1570796309,232.728036403656,32600.885157108307,21927,ellie,10.0.0.158,32600.885157108307,8220000,137,2,4dce3444,5.373340340828945,1.4876673651150072,0.8469675405538404,8206848,8220000,222816.089,10.372,5239.512,4.575,53.87289156626507,51.14397590361447,1.5046327914733034e-37,4.999999873689376e-05,346.3367004394531,0.0006198254995979369,346.3360595703125,-7.386900424677378e-09,0.005078369751572609,-1.3735880851745605,0.0 +3015.6982756844022,1244.613686736468,2146.885572988166,3000.0,20,60000,False,8280000,2760,138,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-22-17,1570796537,228.23901748657227,32829.12417459488,21927,ellie,10.0.0.158,32829.12417459488,8280000,138,2,4dce3444,5.371757589051944,1.4872763627044572,0.846683041768772,8266752,8280000,222797.616,10.327,5245.298,4.607,51.53588957055215,51.46441717791413,7.523163957366517e-38,4.999999873689376e-05,348.5110168457031,-3.231132041037199e-06,348.5110168457031,-2.9292881453812925e-09,0.006246337667107582,-1.3953640460968018,0.0 +3015.6982756844022,1244.613686736468,2186.0758802866053,3000.0,20,60000,False,8340000,2780,139,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-26-04,1570796764,227.13796734809875,33056.26214194298,21927,ellie,10.0.0.158,33056.26214194298,8340000,139,2,4dce3444,5.370249880649552,1.4868848406184705,0.8464006353574385,8326656,8340000,222542.751,10.331,5256.068,4.599,51.298456790123446,50.5993827160494,3.7615819786832586e-38,4.999999873689376e-05,358.5067443847656,-0.00015637304750271142,358.5068664550781,-2.5472071674137453e-10,0.005731170065701008,-1.414859652519226,0.0 +3015.6982756844022,1244.613686736468,2260.8548271805503,3000.0,20,60000,False,8400000,2800,140,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-29-51,1570796991,227.15953612327576,33283.421678066254,21927,ellie,10.0.0.158,33283.421678066254,8400000,140,2,4dce3444,5.368877929898831,1.4864768328432278,0.8461183979887643,8386560,8400000,222580.937,10.301,5260.938,4.604,51.38395061728396,50.63796296296295,1.8807909893416293e-38,4.999999873689376e-05,368.90045166015625,0.0002921150007750839,368.9001159667969,-1.0188828669654981e-09,0.00673392228782177,-1.43276047706604,0.0 +3015.6982756844022,1244.613686736468,2249.04411014296,3000.0,20,60000,False,8460000,2820,141,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-33-39,1570797219,228.06814575195312,33511.48982381821,21927,ellie,10.0.0.158,33511.48982381821,8460000,141,2,4dce3444,5.367493266384629,1.4860623085120963,0.8458380023414043,8446464,8460000,222650.211,10.307,5268.94,4.657,51.26123076923077,50.695692307692305,0.0,4.999999873689376e-05,355.1369323730469,7.804362394381315e-05,355.1368103027344,-3.4387295233528903e-09,0.006698769051581621,-1.439775824546814,0.0 +3015.6982756844022,1270.7331095516838,2243.260530907779,3000.0,20,60000,False,8520000,2840,142,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-37-26,1570797446,226.11893606185913,33737.608759880066,21927,ellie,10.0.0.158,33737.608759880066,8520000,142,2,4dce3444,5.365890332094778,1.4855820354388183,0.8455254971750367,8506368,8520000,222765.244,10.303,5272.515,4.686,51.33034055727554,50.699999999999996,0.0,4.999999873689376e-05,327.8664855957031,0.0005435719504021108,327.86590576171875,-5.858576290762585e-09,0.0040824743919074535,-1.4202516078948975,0.0 +3008.4574808206835,1515.5315443793047,2245.5672120196114,3000.0,20,60000,False,8580000,2860,143,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-41-13,1570797673,227.32266759872437,33964.93142747879,21927,ellie,10.0.0.158,33964.93142747879,8580000,143,2,4dce3444,5.364274087518831,1.4851106844626492,0.8452065279797571,8566272,8580000,222676.693,10.262,5277.496,4.669,51.56481481481482,50.69938271604938,0.0,4.999999873689376e-05,348.1716613769531,-0.000261071661952883,348.17193603515625,-1.528324244937096e-09,0.006652958691120148,-1.4362887144088745,0.0 +2991.6470809203524,1369.3448273987433,2207.9572412199586,3000.0,20,60000,False,8640000,2880,144,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-45-00,1570797900,227.04129576683044,34191.97272324562,21927,ellie,10.0.0.158,34191.97272324562,8640000,144,2,4dce3444,5.362645457430816,1.4847065200354976,0.8448974375852236,8626176,8640000,222771.126,10.226,5283.586,4.642,51.69012345679013,50.70987654320987,0.0,4.999999873689376e-05,300.0270080566406,-6.22580701019615e-05,300.0270690917969,-7.514261213259488e-09,0.0059175798669457436,-1.4059268236160278,0.0 +2991.6470809203524,1358.8131262128595,2179.6720741989375,3000.0,20,60000,False,8700000,2900,145,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-48-47,1570798127,226.96924018859863,34418.94196343422,21927,ellie,10.0.0.158,34418.94196343422,8700000,145,2,4dce3444,5.360991480309143,1.4843293010526113,0.8445919415356835,8686080,8700000,222825.042,10.273,5290.188,4.638,51.28641975308642,50.72716049382716,0.0,4.999999873689376e-05,333.3490905761719,0.00036329388967715204,333.3487548828125,-1.2608674992975466e-08,0.005710863973945379,-1.4260759353637695,0.0 +2991.6470809203524,1358.8131262128595,2159.994130040973,3000.0,20,60000,False,8760000,2920,146,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-52-35,1570798355,228.14227557182312,34647.08423900604,21927,ellie,10.0.0.158,34647.08423900604,8760000,146,2,4dce3444,5.359318093317771,1.483982251462035,0.8442901115009624,8745984,8760000,222775.209,10.11,5089.162,4.568,51.09600000000001,50.79538461538461,0.0,4.999999873689376e-05,330.0295715332031,-0.00018976615683641285,330.02978515625,-5.349134912790987e-09,0.007420304697006941,-1.4448356628417969,0.0 +3029.8487228170975,1358.8131262128595,2187.1887592519556,3000.0,20,60000,False,8820000,2940,147,fec93e2e40d94966abd856ec126a6b85,2019-10-11_05-56-22,1570798582,226.83327293395996,34873.91751194,21927,ellie,10.0.0.158,34873.91751194,8820000,147,2,4dce3444,5.357695230678243,1.4836457334861972,0.8439900268585293,8805888,8820000,222187.694,9.992,5087.422,4.52,51.35864197530864,50.79907407407408,0.0,4.999999873689376e-05,373.81109619140625,0.0008763225050643086,373.8102111816406,-7.1321797356915795e-09,0.007244299631565809,-1.4906598329544067,0.0 +3029.8487228170975,1358.8131262128595,2218.2401870087388,3000.0,20,60000,False,8880000,2960,148,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-00-11,1570798811,229.05873894691467,35102.97625088692,21927,ellie,10.0.0.158,35102.97625088692,8880000,148,2,4dce3444,5.356159607516052,1.4832842715085084,0.8437037015267052,8865792,8880000,222271.767,10.051,5085.31,4.454,51.36697247706422,50.80061162079512,0.0,4.999999873689376e-05,393.29339599609375,0.000779160181991756,393.2926025390625,-3.5660898678457897e-09,0.007883390411734581,-1.523728847503662,0.0 +3057.5341493160295,1358.8131262128595,2230.671498066657,3000.0,20,60000,False,8940000,2980,149,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-03-58,1570799038,226.6279480457306,35329.60419893265,21927,ellie,10.0.0.158,35329.60419893265,8940000,149,2,4dce3444,5.354651983044246,1.482876203436275,0.8434208871911432,8925696,8940000,222224.385,10.001,5081.766,4.43,51.369969040247675,50.86470588235293,0.0,4.999999873689376e-05,315.65411376953125,0.0003467268543317914,315.6537780761719,1.2736035837068727e-10,0.007142008747905493,-1.4739017486572266,0.0 +3057.5341493160295,1474.7052670853525,2214.1094697920144,3000.0,20,60000,False,9000000,3000,150,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-07-44,1570799264,226.70096349716187,35556.30516242981,21927,ellie,10.0.0.158,35556.30516242981,9000000,150,2,4dce3444,5.353170612461417,1.4824547609971188,0.8431499035155278,8985600,9000000,222178.792,10.04,5081.559,4.378,51.32074303405573,50.90092879256965,0.0,4.999999873689376e-05,306.48236083984375,0.000351724709616974,306.48199462890625,-5.349134912790987e-09,0.0059328218922019005,-1.4586695432662964,0.0 +3057.5341493160295,1474.7052670853525,2224.105571036483,3000.0,20,60000,False,9060000,3020,151,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-11-32,1570799492,227.88481163978577,35784.189974069595,21927,ellie,10.0.0.158,35784.189974069595,9060000,151,2,4dce3444,5.3517558760098,1.4820418964062947,0.8428896014343025,9045504,9060000,222162.545,10.816,5078.772,4.303,51.54769230769231,50.91107692307692,0.0,4.999999873689376e-05,344.5024719238281,-0.0006586529780179262,344.503173828125,-7.514261213259488e-09,0.007243610452860594,-1.5162338018417358,0.0 +3057.5341493160295,1474.7052670853525,2224.1196414428223,3000.0,20,60000,False,9120000,3040,152,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-15-26,1570799726,233.78085684776306,36017.97083091736,21927,ellie,10.0.0.158,36017.97083091736,9120000,152,2,4dce3444,5.350540867258457,1.481759550314748,0.8426615768918236,9105408,9120000,222929.011,10.851,5078.523,4.228,54.374251497006,51.065568862275455,0.0,4.999999873689376e-05,368.3914794921875,0.0002147915802197531,368.39129638671875,-6.368017668734183e-09,0.007881799712777138,-1.565125823020935,0.0 +3057.5341493160295,1474.7052670853525,2218.8924527032464,3000.0,20,60000,False,9180000,3060,153,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-19-16,1570799956,229.9773190021515,36247.94814991951,21927,ellie,10.0.0.158,36247.94814991951,9180000,153,2,4dce3444,5.349301529049694,1.4815420060826594,0.8424302987175157,9165312,9180000,223194.821,10.912,5078.04,4.194,51.66310975609756,52.53079268292682,0.0,4.999999873689376e-05,388.4996032714844,0.00033031264320015907,388.499267578125,-8.533143969202683e-09,0.0042679812759160995,-1.571805477142334,0.0 +3064.9934679367625,1474.7052670853525,2289.7909737898376,3000.0,20,60000,False,9240000,3080,154,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-23-05,1570800185,228.6989300251007,36476.64707994461,21927,ellie,10.0.0.158,36476.64707994461,9240000,154,2,4dce3444,5.348167991504749,1.4813427821833463,0.8421961823977767,9225216,9240000,223359.128,10.937,5079.546,4.096,51.42607361963191,55.227607361963194,0.0,4.999999873689376e-05,433.9513854980469,0.0006739410455338657,433.9507141113281,-8.533143969202683e-09,0.00816893670707941,-1.6056526899337769,0.0 +3064.9934679367625,1353.556173376261,2298.5631719351873,3000.0,20,60000,False,9300000,3100,155,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-26-52,1570800412,227.27215909957886,36703.91923904419,21927,ellie,10.0.0.158,36703.91923904419,9300000,155,2,4dce3444,5.347071149900094,1.4811919368471413,0.8419666703778339,9285120,9300000,223389.762,10.915,5079.215,4.082,51.626543209876544,55.300000000000004,0.0,4.999999873689376e-05,319.53692626953125,-3.49509937223047e-05,319.5369873046875,-2.801927800888393e-09,0.006182338111102581,-1.5216206312179565,0.0 +3064.9934679367625,1353.556173376261,2322.818341828732,3000.0,20,60000,False,9360000,3120,156,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-30-40,1570800640,228.04396176338196,36931.96320080757,21927,ellie,10.0.0.158,36931.96320080757,9360000,156,2,4dce3444,5.346011040367947,1.481033668917287,0.8417336354604362,9345024,9360000,223378.541,10.963,5080.387,4.213,51.50337423312883,55.3708588957055,0.0,4.999999873689376e-05,390.2936706542969,0.0007284220191650093,390.2929382324219,-3.5660898678457897e-09,0.0063081602565944195,-1.5931533575057983,0.0 +3064.9934679367625,1353.556173376261,2306.32606831232,3000.0,20,60000,False,9420000,3140,157,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-34-28,1570800868,228.34850311279297,37160.311703920364,21927,ellie,10.0.0.158,37160.311703920364,9420000,157,2,4dce3444,5.344809688630807,1.4808146650910892,0.8414819457011049,9404928,9420000,223528.995,11.014,5081.321,4.238,51.57692307692308,55.39876923076922,0.0,4.999999873689376e-05,350.3853454589844,0.0009383868891745806,350.3844299316406,-1.1462432114583976e-09,0.0073346407152712345,-1.5803309679031372,0.0 +3064.9934679367625,1353.556173376261,2281.101465814145,3000.0,20,60000,False,9480000,3160,158,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-38-15,1570801095,226.2251172065735,37386.53682112694,21927,ellie,10.0.0.158,37386.53682112694,9480000,158,2,4dce3444,5.343573569809669,1.4805231320090535,0.8412314575469421,9464832,9480000,223243.398,10.972,5083.566,4.301,51.342414860681124,55.42724458204334,0.0,4.999999873689376e-05,354.1051025390625,-0.0002704828802961856,354.10540771484375,-1.171715258152517e-08,0.008633880876004696,-1.6004753112792969,0.0 +3029.004997205949,1353.556173376261,2235.2487572029268,3000.0,20,60000,False,9540000,3180,159,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-42-02,1570801322,227.34941363334656,37613.886234760284,21927,ellie,10.0.0.158,37613.886234760284,9540000,159,2,4dce3444,5.342278227971353,1.4802304951242053,0.8409836074254764,9524736,9540000,223312.729,11.003,5086.304,4.303,51.23446153846154,55.49938461538461,0.0,4.999999873689376e-05,358.5614929199219,0.00045807138667441905,358.56103515625,-9.806747414131678e-09,0.006831216160207987,-1.613525629043579,0.0 +3042.2847514446726,1451.9859092532047,2276.989349366336,3000.0,20,60000,False,9600000,3200,160,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-45-51,1570801551,228.53584671020508,37842.42208147049,21927,ellie,10.0.0.158,37842.42208147049,9600000,160,2,4dce3444,5.34102737516111,1.4799183064437402,0.8407323873912261,9584640,9600000,223496.828,10.953,5085.68,4.35,51.33558282208588,55.37208588957056,0.0,4.999999873689376e-05,381.3824462890625,0.0008727480308152735,381.3816223144531,-1.528324244937096e-09,0.0076065571047365665,-1.653287410736084,0.0 +3042.2847514446726,1451.9859092532047,2271.415351346707,3000.0,20,60000,False,9660000,3220,161,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-49-40,1570801780,229.7768681049347,38072.198949575424,21927,ellie,10.0.0.158,38072.198949575424,9660000,161,2,4dce3444,5.339839717209313,1.4796208563731312,0.8404962201768873,9644544,9660000,223685.657,10.262,5086.67,4.402,52.079510703363916,55.34342507645258,0.0,4.999999873689376e-05,376.97265625,0.0006709249573759735,376.9719543457031,-5.603855601776786e-09,0.008511237800121307,-1.6616114377975464,0.0 +3096.7730127033965,1451.9859092532047,2279.3526997793947,3000.0,20,60000,False,9720000,3240,162,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-53-28,1570802008,227.95323491096497,38300.15218448639,21927,ellie,10.0.0.158,38300.15218448639,9720000,162,2,4dce3444,5.338644490268535,1.4793028722546813,0.8402626793614323,9704448,9720000,223101.862,10.258,5087.585,4.518,51.306748466257666,55.3963190184049,0.0,4.999999873689376e-05,366.34429931640625,0.0012409072369337082,366.343017578125,-7.259540080184479e-09,0.00677101407200098,-1.6516884565353394,0.0 +3096.7730127033965,1417.7181955880533,2292.400165494767,3000.0,20,60000,False,9780000,3260,163,fec93e2e40d94966abd856ec126a6b85,2019-10-11_06-57-16,1570802236,228.0283966064453,38528.180581092834,21927,ellie,10.0.0.158,38528.180581092834,9780000,163,2,4dce3444,5.337476606179142,1.479024686978564,0.8400321592535793,9764352,9780000,222907.432,10.21,5087.195,4.566,51.25846153846154,55.42584615384614,0.0,4.999999873689376e-05,376.63421630859375,0.0018430943600833416,376.63238525390625,-9.042585347174281e-09,0.007729583885520697,-1.672771692276001,0.0 +3096.7730127033965,1417.7181955880533,2331.6083649641278,3000.0,20,60000,False,9840000,3280,164,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-01-07,1570802467,230.34946846961975,38758.530049562454,21927,ellie,10.0.0.158,38758.530049562454,9840000,164,2,4dce3444,5.3363840540295975,1.4787958578956728,0.8398038808912065,9824256,9840000,223073.074,10.174,5086.576,4.623,51.208841463414636,55.4844512195122,0.0,4.999999873689376e-05,421.5362548828125,0.0011985857272520661,421.5350646972656,-7.386900424677378e-09,0.006897140759974718,-1.7264200448989868,0.0 +3096.7730127033965,1381.795146883838,2322.2649012357447,3000.0,20,60000,False,9900000,3300,165,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-04-54,1570802694,227.00873231887817,38985.53878188133,21927,ellie,10.0.0.158,38985.53878188133,9900000,165,2,4dce3444,5.335296025113192,1.4785701696964457,0.8395771600303431,9884160,9900000,223048.272,10.151,5085.105,4.592,51.53981481481482,55.498148148148154,0.0,4.999999873689376e-05,370.12017822265625,0.0008794529130682349,370.11932373046875,-7.514261213259488e-09,0.007923475466668606,-1.6857450008392334,0.0 +3096.7730127033965,1381.795146883838,2324.3306171278286,3000.0,20,60000,False,9960000,3320,166,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-08-42,1570802922,228.37273335456848,39213.9115152359,21927,ellie,10.0.0.158,39213.9115152359,9960000,166,2,4dce3444,5.334173849646387,1.4783422650956652,0.839349903255112,9944064,9960000,223079.679,10.169,5086.565,4.607,51.57392638036809,55.50828220858897,0.0,4.999999873689376e-05,377.6601867675781,0.002230962971225381,377.6579284667969,-1.0188828669654981e-09,0.00923401489853859,-1.7261074781417847,0.0 +3052.176600792995,1381.795146883838,2357.590241327462,3000.0,20,60000,False,10020000,3340,167,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-12-32,1570803152,229.55401587486267,39443.46553111076,21927,ellie,10.0.0.158,39443.46553111076,10020000,167,2,4dce3444,5.333127871085545,1.4781177331046573,0.8391280804940919,10003968,10020000,223203.96,10.192,5082.857,4.615,51.340853658536595,55.59603658536587,0.0,4.999999873689376e-05,409.90966796875,0.0011881475802510977,409.908447265625,-2.4198467674096946e-09,0.00955733098089695,-1.7725549936294556,0.0 +3052.176600792995,1381.795146883838,2354.666773378418,3000.0,20,60000,False,10080000,3360,168,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-16-23,1570803383,231.84287452697754,39675.30840563774,21927,ellie,10.0.0.158,39675.30840563774,10080000,168,2,4dce3444,5.332240199268739,1.4779118357968475,0.8389328324412136,10063872,10080000,223768.458,10.234,5080.151,4.531,54.032424242424234,55.720606060606066,0.0,4.999999873689376e-05,369.3013610839844,0.0004047710681334138,369.3009338378906,-8.787864658188482e-09,0.008358349092304707,-1.7276036739349365,0.0 +3052.176600792995,1381.795146883838,2327.8122927546174,3000.0,20,60000,False,10140000,3380,169,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-20-11,1570803611,227.99224424362183,39903.30064988136,21927,ellie,10.0.0.158,39903.30064988136,10140000,169,2,4dce3444,5.331351283884022,1.4776509727492602,0.8387472494796107,10123776,10140000,223834.355,10.395,5078.363,4.598,51.76779141104296,53.49233128834355,0.0,4.999999873689376e-05,380.6027526855469,0.0008131205104291439,380.6019287109375,-6.368017779756485e-10,0.006627724505960941,-1.7412049770355225,0.0 +3052.176600792995,1552.5969516358923,2331.345813736235,3000.0,20,60000,False,10200000,3400,170,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-23-59,1570803839,227.18774437904358,40130.48839426041,21927,ellie,10.0.0.158,40130.48839426041,10200000,170,2,4dce3444,5.330468666838864,1.4773921399831624,0.8385629159773883,10183680,10200000,223699.368,10.394,5078.525,4.653,51.51574074074074,51.41635802469135,0.0,4.999999873689376e-05,382.2425231933594,0.0008789993589743972,382.24163818359375,-3.948171123369093e-09,0.007721083704382181,-1.7556211948394775,0.0 +3041.4282666399654,1552.5969516358923,2335.5679607765587,3000.0,20,60000,False,10260000,3420,171,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-27-47,1570804067,228.18240404129028,40358.6707983017,21927,ellie,10.0.0.158,40358.6707983017,10260000,171,2,4dce3444,5.329592778609273,1.4771153921602573,0.8383746656545878,10243584,10260000,223542.754,10.309,5075.766,4.68,51.366769230769236,51.49292307692307,0.0,4.999999873689376e-05,387.7888488769531,0.0009865574538707733,387.7878723144531,-1.273603555951297e-09,0.0071579329669475555,-1.789628028869629,0.0 +3041.277396390285,1552.5969516358923,2291.3922025148477,3000.0,20,60000,False,10320000,3440,172,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-31-35,1570804295,227.84405541419983,40586.5148537159,21927,ellie,10.0.0.158,40586.5148537159,10320000,172,2,4dce3444,5.328647421407559,1.4768448623325172,0.8381834846618813,10303488,10320000,223531.833,10.324,5075.787,4.652,51.2883076923077,51.49230769230769,0.0,4.999999873689376e-05,348.61578369140625,0.00047967577120289207,348.6153259277344,-3.8208107788761936e-09,0.009670348837971687,-1.752366304397583,0.0 +3054.637216163588,1570.1410214240584,2285.7422633450497,3000.0,20,60000,False,10380000,3460,173,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-35-23,1570804523,228.62696480751038,40815.14181852341,21927,ellie,10.0.0.158,40815.14181852341,10380000,173,2,4dce3444,5.327610210989354,1.4765507393155133,0.8379799893952782,10363392,10380000,223596.012,10.306,5071.499,4.609,51.26901840490799,51.48312883435583,0.0,4.999999873689376e-05,360.74285888671875,0.0010298284469172359,360.7418518066406,-5.603855601776786e-09,0.006654420401901007,-1.7767667770385742,0.0 +3054.637216163588,1571.5596343299374,2307.4846784830906,3000.0,20,60000,False,10440000,3480,174,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-39-14,1570804754,230.36347889900208,41045.50529742241,21927,ellie,10.0.0.158,41045.50529742241,10440000,174,2,4dce3444,5.326640062500885,1.4763131403663496,0.8377827494028618,10423296,10440000,223599.487,10.318,5069.426,4.637,51.52857142857143,51.50516717325228,0.0,4.999999873689376e-05,412.6548767089844,0.0013235356891527772,412.653564453125,-5.7312159462696854e-09,0.01018442865461111,-1.8456313610076904,0.0 +3054.637216163588,1617.8097711668156,2309.134063394056,3000.0,20,60000,False,10500000,3500,175,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-43-00,1570804980,226.58526182174683,41272.090559244156,21927,ellie,10.0.0.158,41272.090559244156,10500000,175,2,4dce3444,5.325675384201543,1.476061597471345,0.8375878027438316,10483200,10500000,223556.375,10.336,5070.112,4.665,51.372445820433434,51.51021671826625,0.0,4.999999873689376e-05,384.08416748046875,0.0017204862087965012,384.0824279785156,-8.405783624709784e-09,0.00944794062525034,-1.7994898557662964,0.0 +3054.637216163588,1617.8097711668156,2325.1215299217442,3000.0,20,60000,False,10560000,3520,176,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-46-50,1570805210,229.45271515846252,41501.54327440262,21927,ellie,10.0.0.158,41501.54327440262,10560000,176,2,4dce3444,5.3247633479586,1.4758386395361094,0.8373961014041721,10543104,10560000,223663.784,10.295,5070.802,4.616,51.712804878048786,51.591463414634156,0.0,4.999999873689376e-05,413.54815673828125,0.0012772162444889545,413.546875,-4.839693534819389e-09,0.009891820140182972,-1.8770949840545654,0.0 +3054.637216163588,1617.8097711668156,2351.552939156018,3000.0,20,60000,False,10620000,3540,177,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-50-38,1570805438,228.038245677948,41729.58152008057,21927,ellie,10.0.0.158,41729.58152008057,10620000,177,2,4dce3444,5.323892085106135,1.4756077989050795,0.8372065220748831,10603008,10620000,223506.598,10.243,5076.401,4.617,51.31846153846154,51.599076923076936,0.0,4.999999873689376e-05,391.6576232910156,0.0014252109685912728,391.6562194824219,-5.094414223805188e-09,0.009858991019427776,-1.8501771688461304,0.0 +3037.2034171631726,1379.466660685787,2387.840205469589,3000.0,20,60000,False,10680000,3560,178,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-54-27,1570805667,229.4220325946808,41959.00355267525,21927,ellie,10.0.0.158,41959.00355267525,10680000,178,2,4dce3444,5.323056324836562,1.4753834938093247,0.8370115137179864,10662912,10680000,223262.66,10.254,5078.192,4.713,51.26085626911315,51.62629969418961,0.0,4.999999873689376e-05,418.8285827636719,0.0018733737524598837,418.8267517089844,-2.9292881453812925e-09,0.010292772203683853,-1.896974802017212,0.0 +3027.3897033104518,1379.466660685787,2351.7860060329613,3000.0,20,60000,False,10740000,3580,179,fec93e2e40d94966abd856ec126a6b85,2019-10-11_07-58-14,1570805894,227.16819834709167,42186.17175102234,21927,ellie,10.0.0.158,42186.17175102234,10740000,179,2,4dce3444,5.32213661438874,1.4751260472793246,0.8368124718118618,10722816,10740000,223182.238,10.078,5076.401,4.718,51.42407407407408,51.69475308641975,0.0,4.999999873689376e-05,353.7749328613281,0.0018383120186626911,353.77313232421875,-1.2099233615003868e-08,0.007688155397772789,-1.8460592031478882,0.0 +3027.3897033104518,1379.466660685787,2333.6047019246557,3000.0,20,60000,False,10800000,3600,180,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-02-02,1570806122,227.72692155838013,42413.89867258072,21927,ellie,10.0.0.158,42413.89867258072,10800000,180,2,4dce3444,5.321248849609199,1.4748735382687588,0.8366240309396852,10782720,10800000,223234.743,10.053,5077.939,4.6,51.39507692307693,51.699384615384616,0.0,4.999999873689376e-05,350.7579650878906,0.0006976635777391493,350.75726318359375,-4.967053879312289e-09,0.010407296940684319,-1.8338757753372192,0.0 +3027.3897033104518,1379.466660685787,2309.0435616183545,3000.0,20,60000,False,10860000,3620,181,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-05-49,1570806349,227.1251826286316,42641.02385520935,21927,ellie,10.0.0.158,42641.02385520935,10860000,181,2,4dce3444,5.32027988142525,1.4745946199647024,0.8364303488692733,10842624,10860000,223125.729,10.101,5081.137,4.595,51.2641975308642,51.75833333333333,0.0,4.999999873689376e-05,373.5103759765625,0.0015793710481375456,373.5087890625,-4.58497284583359e-09,0.009842636995017529,-1.8823915719985962,0.0 +3027.3897033104518,1379.466660685787,2333.5278943553653,3000.0,20,60000,False,10920000,3640,182,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-09-40,1570806580,230.62169885635376,42871.645554065704,21927,ellie,10.0.0.158,42871.645554065704,10920000,182,2,4dce3444,5.319407906525791,1.4743501908323235,0.836247519749561,10902528,10920000,223405.266,10.035,5079.476,4.6,51.54924012158056,51.79574468085107,0.0,4.999999873689376e-05,425.79461669921875,0.001901145908050239,425.792724609375,-3.8208107788761936e-09,0.010114246979355812,-1.959778904914856,0.0 +3016.9403554476085,1632.9454111848383,2312.9844463865734,3000.0,20,60000,False,10980000,3660,183,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-13-29,1570806809,228.80314350128174,43100.448697566986,21927,ellie,10.0.0.158,43100.448697566986,10980000,183,2,4dce3444,5.31852330585841,1.4741123660214692,0.8360715576549578,10962432,10980000,223419.137,10.077,5083.09,4.654,51.340797546012276,51.79877300613497,0.0,4.999999873689376e-05,385.3238220214844,0.0021609573159366846,385.3216552734375,-6.750098702212881e-09,0.012110111303627491,-1.9301774501800537,0.0 +3081.135910454933,1632.9454111848383,2356.752828140425,3000.0,20,60000,False,11040000,3680,184,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-17-21,1570807041,232.6968982219696,43333.145595788956,21927,ellie,10.0.0.158,43333.145595788956,11040000,184,2,4dce3444,5.317806953497146,1.4739050282046844,0.835913804072241,11022336,11040000,223653.408,10.053,5082.151,4.676,53.984939759036145,51.96596385542169,0.0,4.999999873689376e-05,433.0020751953125,0.0017846066039055586,433.00030517578125,-4.202891812354892e-09,0.009497447870671749,-1.9684594869613647,0.0 +3081.135910454933,1667.394458938387,2381.375885498674,3000.0,20,60000,False,11100000,3700,185,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-21-11,1570807271,229.64303302764893,43562.788628816605,21927,ellie,10.0.0.158,43562.788628816605,11100000,185,2,4dce3444,5.317152155339651,1.4737214327130448,0.835760440683614,11082240,11100000,223959.642,10.005,5081.88,4.57,51.84054878048781,55.32225609756098,0.0,4.999999873689376e-05,392.8614807128906,0.0022623285185545683,392.8591613769531,-4.202891812354892e-09,0.007646601647138596,-1.9432039260864258,0.0 +3098.8350768758964,1625.2171266307498,2379.2991182695537,3000.0,20,60000,False,11160000,3720,186,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-25-00,1570807500,229.00459384918213,43791.79322266579,21927,ellie,10.0.0.158,43791.79322266579,11160000,186,2,4dce3444,5.31657170797969,1.4735733613793303,0.8356162438051601,11142144,11160000,223915.74,10.021,5080.988,4.505,51.5743119266055,55.998165137614684,0.0,4.999999873689376e-05,383.97369384765625,0.0026963071431964636,383.97100830078125,-5.4764952572838865e-09,0.009138607420027256,-1.9157949686050415,0.0 +3098.8350768758964,1625.2171266307498,2351.530405847605,3000.0,20,60000,False,11220000,3740,187,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-28-50,1570807730,229.3536684513092,44021.146891117096,21927,ellie,10.0.0.158,44021.146891117096,11220000,187,2,4dce3444,5.31592791592413,1.4734123641842751,0.8354681170802638,11202048,11220000,224050.321,10.052,5078.057,4.427,51.22660550458715,56.01070336391438,0.0,4.999999873689376e-05,384.4533996582031,0.0019912065472453833,384.4513854980469,-1.273603555951297e-09,0.011389504186809063,-1.958509922027588,0.0 +3098.8350768758964,1625.2171266307498,2381.9223338432475,3000.0,20,60000,False,11280000,3760,188,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-32-38,1570807958,228.32455682754517,44249.47144794464,21927,ellie,10.0.0.158,44249.47144794464,11280000,188,2,4dce3444,5.315320223039513,1.4732345124600834,0.8353162588568841,11261952,11280000,223943.815,10.064,5074.833,4.365,51.424233128834366,56.09263803680983,0.0,4.999999873689376e-05,435.67926025390625,0.0012859683483839035,435.6779479980469,-5.4764952572838865e-09,0.009204023517668247,-2.0398941040039062,0.0 +3098.8350768758964,1625.2171266307498,2345.661368938575,3000.0,20,60000,False,11340000,3780,189,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-36-26,1570808186,228.5127408504486,44477.98418879509,21927,ellie,10.0.0.158,44477.98418879509,11340000,189,2,4dce3444,5.314600932128668,1.473040611884727,0.8351520134882991,11321856,11340000,224075.614,10.103,5077.491,4.275,51.38865030674847,56.099693251533765,0.0,4.999999873689376e-05,373.9120788574219,0.0016731752548366785,373.910400390625,3.82081061234274e-10,0.012111539952456951,-1.958390235900879,0.0 +3098.8350768758964,1625.2171266307498,2342.3243566614283,3000.0,20,60000,False,11400000,3800,190,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-40-16,1570808416,229.3007504940033,44707.28493928909,21927,ellie,10.0.0.158,44707.28493928909,11400000,190,2,4dce3444,5.313873394351817,1.4728552651667477,0.8349895846908116,11381760,11400000,224234.021,10.158,5076.36,4.321,51.472477064220186,56.148929663608556,0.0,4.999999873689376e-05,388.7669982910156,0.004370197653770447,388.7625732421875,-1.0061468103117477e-08,0.014821565710008144,-1.9951106309890747,0.0 +3054.7912484319218,1644.044391956315,2351.491244229064,3000.0,20,60000,False,11460000,3820,191,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-44-05,1570808645,229.0536184310913,44936.338557720184,21927,ellie,10.0.0.158,44936.338557720184,11460000,191,2,4dce3444,5.313113009251372,1.4726607383911072,0.8348284049296928,11441664,11460000,224428.723,10.14,5074.62,4.229,51.11042944785276,56.1638036809816,0.0,4.999999873689376e-05,394.8280029296875,0.002570396289229393,394.825439453125,-1.1971873270510969e-08,0.012304340489208698,-2.016383409500122,0.0 +3097.0499886949797,1644.044391956315,2380.7444319412434,3000.0,20,60000,False,11520000,3840,192,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-47-55,1570808875,229.77351784706116,45166.112075567245,21927,ellie,10.0.0.158,45166.112075567245,11520000,192,2,4dce3444,5.312370471447091,1.4724729868923594,0.8346652715441004,11501568,11520000,224340.091,10.19,5078.363,4.206,51.34634146341464,56.01493902439025,0.0,4.999999873689376e-05,433.66241455078125,0.002431400353088975,433.6600036621094,-4.457612501340691e-09,0.012421557679772377,-2.087833881378174,0.0 +3097.0499886949797,1644.044391956315,2368.369046283986,3000.0,20,60000,False,11580000,3860,193,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-51-44,1570809104,229.3771834373474,45395.48925900459,21927,ellie,10.0.0.158,45395.48925900459,11580000,193,2,4dce3444,5.311642060199249,1.4722870027966446,0.8345068242874324,11561472,11580000,224396.483,10.2,5079.43,4.198,51.203058103975536,56.08685015290523,0.0,4.999999873689376e-05,420.4528503417969,0.0016009387327358127,420.45123291015625,-6.240657324241283e-09,0.01214947272092104,-2.0898144245147705,0.0 +3097.0499886949797,1537.9424905465285,2424.6104230103965,3000.0,20,60000,False,11640000,3880,194,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-55-35,1570809335,231.52236151695251,45627.011620521545,21927,ellie,10.0.0.158,45627.011620521545,11640000,194,2,4dce3444,5.311017146456566,1.4721219164898418,0.8343568640810725,11621376,11640000,224276.656,10.257,5081.77,4.139,51.57885196374622,56.09848942598189,0.0,4.999999873689376e-05,466.02496337890625,0.0033219954930245876,466.0216369628906,-4.58497284583359e-09,0.016518140211701393,-2.150561809539795,0.0 +3097.0499886949797,1537.9424905465285,2421.0279043025266,3000.0,20,60000,False,11700000,3900,195,fec93e2e40d94966abd856ec126a6b85,2019-10-11_08-59-23,1570809563,227.08076214790344,45854.09238266945,21927,ellie,10.0.0.158,45854.09238266945,11700000,195,2,4dce3444,5.310349665648566,1.4719305297110157,0.8341999021198612,11681280,11700000,224016.541,10.279,5085.547,4.246,51.35679012345678,55.60061728395061,0.0,4.999999873689376e-05,389.2752685546875,0.0028628669679164886,389.2723693847656,-8.915225002681382e-09,0.012709049507975578,-2.0523343086242676,0.0 +3097.0499886949797,1537.9424905465285,2409.73189234466,3000.0,20,60000,False,11760000,3920,196,fec93e2e40d94966abd856ec126a6b85,2019-10-11_09-03-11,1570809791,228.6441512107849,46082.736533880234,21927,ellie,10.0.0.158,46082.736533880234,11760000,196,2,4dce3444,5.309671688070273,1.4717520710850662,0.8340386012548988,11741184,11760000,223978.794,10.278,5087.212,4.308,51.12147239263804,54.19907975460123,0.0,4.999999873689376e-05,383.9013671875,0.0027107472997158766,383.8986511230469,0.0,0.012237018905580044,-2.0226101875305176,0.0 +3067.8270147093335,1537.9424905465285,2373.2781452362055,3000.0,20,60000,False,11820000,3940,197,fec93e2e40d94966abd856ec126a6b85,2019-10-11_09-07-00,1570810020,229.27349042892456,46312.01002430916,21927,ellie,10.0.0.158,46312.01002430916,11820000,197,2,4dce3444,5.309009505881488,1.4715678069903064,0.8338863915694975,11801088,11820000,223971.878,10.263,5086.107,4.304,51.42752293577981,54.199999999999996,0.0,4.999999873689376e-05,381.7317810058594,0.001686975359916687,381.7301025390625,-1.7830449339228949e-09,0.008802078664302826,-2.056385040283203,0.0 +3067.8270147093335,1537.9424905465285,2362.2314232442523,3000.0,20,60000,False,11880000,3960,198,fec93e2e40d94966abd856ec126a6b85,2019-10-11_09-10-50,1570810250,229.05612421035767,46541.066148519516,21927,ellie,10.0.0.158,46541.066148519516,11880000,198,2,4dce3444,5.308319224551878,1.47140686606288,0.8337354392680725,11860992,11880000,224042.792,10.249,5088.39,4.227,51.28348623853211,54.2217125382263,0.0,4.999999873689376e-05,409.4963073730469,0.0022892404813319445,409.4940185546875,-6.8774590467057806e-09,0.012833282351493835,-2.0907657146453857,0.0 +3067.8270147093335,1617.0572138047326,2309.6644856533026,3000.0,20,60000,False,11940000,3980,199,fec93e2e40d94966abd856ec126a6b85,2019-10-11_09-14-39,1570810479,229.57902789115906,46770.645176410675,21927,ellie,10.0.0.158,46770.645176410675,11940000,199,2,4dce3444,5.307533921500677,1.4712477307166443,0.833577081298144,11920896,11940000,224030.812,10.206,5207.114,4.161,52.2617737003058,54.309174311926604,0.0,4.999999873689376e-05,391.09295654296875,0.0038364955689758062,391.0890808105469,-7.641621557752387e-09,0.013565467670559883,-2.065732955932617,0.0 +3067.8270147093335,1384.8900407828025,2324.433615775931,3000.0,20,60000,True,12000000,4000,200,fec93e2e40d94966abd856ec126a6b85,2019-10-11_09-18-33,1570810713,233.92884373664856,47004.57402014732,21927,ellie,10.0.0.158,47004.57402014732,12000000,200,2,4dce3444,5.306923250069895,1.4711140704663632,0.8334424370031818,11980800,12000000,224493.937,10.2,5206.619,4.328,53.853592814371254,54.06766467065868,0.0,4.999999873689376e-05,414.9051818847656,0.0038591448683291674,414.9013366699219,-6.240657324241283e-09,0.01615159772336483,-2.1259605884552,0.0 diff --git a/tutorials/data/trained_ring/result.json b/tutorials/data/trained_ring/result.json new file mode 100644 index 000000000..ce15c04c4 --- /dev/null +++ b/tutorials/data/trained_ring/result.json @@ -0,0 +1,200 @@ +{"episode_reward_max": -7025.808506814366, "episode_reward_min": -9261.36558597124, "episode_reward_mean": -8112.181357017001, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.088170867479468, "mean_processing_ms": 1.4314549290566863, "mean_inference_ms": 0.7895048544933859}, "off_policy_estimator": {}, "info": {"num_steps_trained": 59904, "num_steps_sampled": 60000, "sample_time_ms": 222168.726, "load_time_ms": 70.708, "grad_time_ms": 5654.924, "update_time_ms": 559.798, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 5841.9599609375, "policy_loss": -0.010422835126519203, "vf_loss": 5841.9677734375, "vf_explained_var": -6.949774251552299e-05, "kl": 0.011039670556783676, "entropy": 1.3680330514907837, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 60000, "episodes_total": 20, "training_iteration": 1, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_20-18-55", "timestamp": 1570763935, "time_this_iter_s": 228.52161312103271, "time_total_s": 228.52161312103271, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 228.52161312103271, "timesteps_since_restore": 60000, "iterations_since_restore": 1, "perf": {"cpu_util_percent": 52.21559633027523, "ram_util_percent": 71.73363914373088}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": -5906.2140718487135, "episode_reward_min": -9261.36558597124, "episode_reward_mean": -7658.758736087065, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.0538187367921905, "mean_processing_ms": 1.4290795645209107, "mean_inference_ms": 0.7888463681015899}, "off_policy_estimator": {}, "info": {"num_steps_trained": 119808, "num_steps_sampled": 120000, "sample_time_ms": 219066.136, "load_time_ms": 40.976, "grad_time_ms": 5381.374, "update_time_ms": 282.127, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 4593.02490234375, "policy_loss": -0.01005244255065918, "vf_loss": 4593.033203125, "vf_explained_var": -4.692286529461853e-05, "kl": 0.010486019775271416, "entropy": 1.318793773651123, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 120000, "episodes_total": 40, "training_iteration": 2, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_20-22-36", "timestamp": 1570764156, "time_this_iter_s": 221.10058522224426, "time_total_s": 449.622198343277, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 449.622198343277, "timesteps_since_restore": 120000, "iterations_since_restore": 2, "perf": {"cpu_util_percent": 51.92380952380953, "ram_util_percent": 71.6184126984127}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": -5013.736690034951, "episode_reward_min": -9261.36558597124, "episode_reward_mean": -7143.2109891870605, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.039747437323658, "mean_processing_ms": 1.428160692276646, "mean_inference_ms": 0.788035512343128}, "off_policy_estimator": {}, "info": {"num_steps_trained": 179712, "num_steps_sampled": 180000, "sample_time_ms": 218389.836, "load_time_ms": 31.292, "grad_time_ms": 5285.999, "update_time_ms": 189.445, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 3316.42529296875, "policy_loss": -0.010575403459370136, "vf_loss": 3316.433837890625, "vf_explained_var": -5.873668487765826e-05, "kl": 0.011085903272032738, "entropy": 1.2667220830917358, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 180000, "episodes_total": 60, "training_iteration": 3, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_20-26-18", "timestamp": 1570764378, "time_this_iter_s": 222.16285920143127, "time_total_s": 671.7850575447083, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 671.7850575447083, "timesteps_since_restore": 180000, "iterations_since_restore": 3, "perf": {"cpu_util_percent": 51.99432176656152, "ram_util_percent": 71.66403785488959}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": -4456.1761653801295, "episode_reward_min": -9261.36558597124, "episode_reward_mean": -6669.404688050863, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.030053462651056, "mean_processing_ms": 1.4308519019395238, "mean_inference_ms": 0.7870829591072147}, "off_policy_estimator": {}, "info": {"num_steps_trained": 239616, "num_steps_sampled": 240000, "sample_time_ms": 218664.778, "load_time_ms": 28.448, "grad_time_ms": 5340.407, "update_time_ms": 143.086, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 2439.493896484375, "policy_loss": -0.011184969916939735, "vf_loss": 2439.502685546875, "vf_explained_var": -2.6020992663688958e-05, "kl": 0.011975754983723164, "entropy": 1.2135356664657593, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 240000, "episodes_total": 80, "training_iteration": 4, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_20-30-03", "timestamp": 1570764603, "time_this_iter_s": 225.03357934951782, "time_total_s": 896.8186368942261, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 896.8186368942261, "timesteps_since_restore": 240000, "iterations_since_restore": 4, "perf": {"cpu_util_percent": 51.666043613707174, "ram_util_percent": 71.84859813084113}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": -3487.5620037295853, "episode_reward_min": -9261.36558597124, "episode_reward_mean": -6321.31384678577, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.044428040388933, "mean_processing_ms": 1.4386925304952547, "mean_inference_ms": 0.7907806502716475}, "off_policy_estimator": {}, "info": {"num_steps_trained": 299520, "num_steps_sampled": 300000, "sample_time_ms": 223430.825, "load_time_ms": 25.437, "grad_time_ms": 5558.887, "update_time_ms": 115.179, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 2172.185302734375, "policy_loss": -0.011577743105590343, "vf_loss": 2172.194091796875, "vf_explained_var": -7.91952152212616e-06, "kl": 0.012583091855049133, "entropy": 1.164360523223877, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 300000, "episodes_total": 100, "training_iteration": 5, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_20-34-12", "timestamp": 1570764852, "time_this_iter_s": 248.96133828163147, "time_total_s": 1145.7799751758575, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 1145.7799751758575, "timesteps_since_restore": 300000, "iterations_since_restore": 5, "perf": {"cpu_util_percent": 58.12845070422535, "ram_util_percent": 61.5149295774648}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": -2854.9588710832304, "episode_reward_min": -8096.150249372029, "episode_reward_mean": -5476.13577015095, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.073097671189653, "mean_processing_ms": 1.4529774357459888, "mean_inference_ms": 0.7989281480451976}, "off_policy_estimator": {}, "info": {"num_steps_trained": 359424, "num_steps_sampled": 360000, "sample_time_ms": 228890.611, "load_time_ms": 23.431, "grad_time_ms": 5704.099, "update_time_ms": 96.898, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 1362.3341064453125, "policy_loss": -0.012156374752521515, "vf_loss": 1362.3438720703125, "vf_explained_var": -2.1534087863983586e-06, "kl": 0.012464698404073715, "entropy": 1.1054127216339111, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 360000, "episodes_total": 120, "training_iteration": 6, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_20-38-35", "timestamp": 1570765115, "time_this_iter_s": 262.6561372280121, "time_total_s": 1408.4361124038696, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 1408.4361124038696, "timesteps_since_restore": 360000, "iterations_since_restore": 6, "perf": {"cpu_util_percent": 59.342666666666666, "ram_util_percent": 45.55626666666667}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": -2221.00948676975, "episode_reward_min": -7149.132676521768, "episode_reward_mean": -4656.074478866747, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.153275956053091, "mean_processing_ms": 1.4770714013878166, "mean_inference_ms": 0.8140585646975877}, "off_policy_estimator": {}, "info": {"num_steps_trained": 419328, "num_steps_sampled": 420000, "sample_time_ms": 236927.77, "load_time_ms": 22.291, "grad_time_ms": 6257.975, "update_time_ms": 83.76, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 870.5482788085938, "policy_loss": -0.011359051801264286, "vf_loss": 870.5573120117188, "vf_explained_var": -7.463316364919592e-07, "kl": 0.011871282942593098, "entropy": 1.0494786500930786, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 420000, "episodes_total": 140, "training_iteration": 7, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_20-43-30", "timestamp": 1570765410, "time_this_iter_s": 294.7735686302185, "time_total_s": 1703.2096810340881, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 1703.2096810340881, "timesteps_since_restore": 420000, "iterations_since_restore": 7, "perf": {"cpu_util_percent": 65.39642857142857, "ram_util_percent": 49.371904761904766}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": -1452.2763886998616, "episode_reward_min": -6322.268008276782, "episode_reward_mean": -3933.3365328160976, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.290933627345668, "mean_processing_ms": 1.5120344816730351, "mean_inference_ms": 0.8402113955101816}, "off_policy_estimator": {}, "info": {"num_steps_trained": 479232, "num_steps_sampled": 480000, "sample_time_ms": 248612.429, "load_time_ms": 21.439, "grad_time_ms": 6501.322, "update_time_ms": 73.941, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 597.2427978515625, "policy_loss": -0.012078477069735527, "vf_loss": 597.2522583007812, "vf_explained_var": -4.884269628746551e-07, "kl": 0.013157332316040993, "entropy": 0.9939395785331726, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 480000, "episodes_total": 160, "training_iteration": 8, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_20-49-08", "timestamp": 1570765748, "time_this_iter_s": 338.6552450656891, "time_total_s": 2041.8649260997772, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 2041.8649260997772, "timesteps_since_restore": 480000, "iterations_since_restore": 8, "perf": {"cpu_util_percent": 69.82111801242236, "ram_util_percent": 53.56438923395445}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": -1118.3189766642274, "episode_reward_min": -6124.9506407198605, "episode_reward_mean": -3307.968411093388, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.480940687804025, "mean_processing_ms": 1.55489631491183, "mean_inference_ms": 0.8770157807184243}, "off_policy_estimator": {}, "info": {"num_steps_trained": 539136, "num_steps_sampled": 540000, "sample_time_ms": 259234.077, "load_time_ms": 20.498, "grad_time_ms": 6839.633, "update_time_ms": 67.039, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 409.6300048828125, "policy_loss": -0.013440164737403393, "vf_loss": 409.6407165527344, "vf_explained_var": -6.151505260731938e-08, "kl": 0.0134345144033432, "entropy": 0.9395018815994263, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 540000, "episodes_total": 180, "training_iteration": 9, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_20-55-02", "timestamp": 1570766102, "time_this_iter_s": 353.79422402381897, "time_total_s": 2395.659150123596, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 2395.659150123596, "timesteps_since_restore": 540000, "iterations_since_restore": 9, "perf": {"cpu_util_percent": 71.38452380952381, "ram_util_percent": 55.07023809523809}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": -810.4557983671579, "episode_reward_min": -4998.431657411379, "episode_reward_mean": -2668.5023948431453, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.659735332467301, "mean_processing_ms": 1.5931577852227674, "mean_inference_ms": 0.9113241213176999}, "off_policy_estimator": {}, "info": {"num_steps_trained": 599040, "num_steps_sampled": 600000, "sample_time_ms": 260963.121, "load_time_ms": 19.964, "grad_time_ms": 6872.463, "update_time_ms": 60.652, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 290.8235778808594, "policy_loss": -0.011462359689176083, "vf_loss": 290.83251953125, "vf_explained_var": -2.2670143096092943e-08, "kl": 0.01251077651977539, "entropy": 0.8885784149169922, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 600000, "episodes_total": 200, "training_iteration": 10, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_20-59-46", "timestamp": 1570766386, "time_this_iter_s": 283.7291479110718, "time_total_s": 2679.388298034668, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 2679.388298034668, "timesteps_since_restore": 600000, "iterations_since_restore": 10, "perf": {"cpu_util_percent": 61.58444444444444, "ram_util_percent": 47.998024691358026}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": -93.82992271840153, "episode_reward_min": -4293.155475649472, "episode_reward_mean": -2089.294018988054, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.821826827737394, "mean_processing_ms": 1.6264132822510564, "mean_inference_ms": 0.9423207702497534}, "off_policy_estimator": {}, "info": {"num_steps_trained": 658944, "num_steps_sampled": 660000, "sample_time_ms": 266770.744, "load_time_ms": 14.156, "grad_time_ms": 7030.347, "update_time_ms": 5.173, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 114.49359130859375, "policy_loss": -0.011447830125689507, "vf_loss": 114.50259399414062, "vf_explained_var": -2.2542781863421624e-08, "kl": 0.012185333296656609, "entropy": 0.8246817588806152, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 660000, "episodes_total": 220, "training_iteration": 11, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-04-34", "timestamp": 1570766674, "time_this_iter_s": 287.5162422657013, "time_total_s": 2966.9045403003693, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 2966.9045403003693, "timesteps_since_restore": 660000, "iterations_since_restore": 11, "perf": {"cpu_util_percent": 61.77024390243902, "ram_util_percent": 47.73658536585366}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 108.44067851910259, "episode_reward_min": -3942.7840997588282, "episode_reward_mean": -1632.5435440635117, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.953380432188706, "mean_processing_ms": 1.6516868168672698, "mean_inference_ms": 0.9678841027984157}, "off_policy_estimator": {}, "info": {"num_steps_trained": 718848, "num_steps_sampled": 720000, "sample_time_ms": 273030.331, "load_time_ms": 14.465, "grad_time_ms": 7234.085, "update_time_ms": 5.306, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 85.70304870605469, "policy_loss": -0.011727341450750828, "vf_loss": 85.71227264404297, "vf_explained_var": -1.1080350859060673e-08, "kl": 0.012510515749454498, "entropy": 0.7761985659599304, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 720000, "episodes_total": 240, "training_iteration": 12, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-09-19", "timestamp": 1570766959, "time_this_iter_s": 285.7416036128998, "time_total_s": 3252.646143913269, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 3252.646143913269, "timesteps_since_restore": 720000, "iterations_since_restore": 12, "perf": {"cpu_util_percent": 61.30098039215686, "ram_util_percent": 48.312254901960785}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 673.5586714703026, "episode_reward_min": -2839.3240533315893, "episode_reward_mean": -1219.8554623415228, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.034394240623488, "mean_processing_ms": 1.6665883136925579, "mean_inference_ms": 0.9837252258844218}, "off_policy_estimator": {}, "info": {"num_steps_trained": 778752, "num_steps_sampled": 780000, "sample_time_ms": 279125.462, "load_time_ms": 14.509, "grad_time_ms": 7438.321, "update_time_ms": 5.279, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 57.19596862792969, "policy_loss": -0.010404279455542564, "vf_loss": 57.20412826538086, "vf_explained_var": -1.5283243115504774e-08, "kl": 0.011229820549488068, "entropy": 0.7205685377120972, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 780000, "episodes_total": 260, "training_iteration": 13, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-14-04", "timestamp": 1570767244, "time_this_iter_s": 285.16099739074707, "time_total_s": 3537.807141304016, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 3537.807141304016, "timesteps_since_restore": 780000, "iterations_since_restore": 13, "perf": {"cpu_util_percent": 60.53029556650246, "ram_util_percent": 49.21108374384237}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 799.0577524733043, "episode_reward_min": -2603.272183032721, "episode_reward_mean": -807.7392234128113, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.071445044177026, "mean_processing_ms": 1.6721416835638423, "mean_inference_ms": 0.9903971013706158}, "off_policy_estimator": {}, "info": {"num_steps_trained": 838656, "num_steps_sampled": 840000, "sample_time_ms": 285426.019, "load_time_ms": 13.711, "grad_time_ms": 7612.328, "update_time_ms": 5.314, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 39.828609466552734, "policy_loss": -0.009519231505692005, "vf_loss": 39.83600997924805, "vf_explained_var": 0.0, "kl": 0.010611079633235931, "entropy": 0.6647995114326477, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 840000, "episodes_total": 280, "training_iteration": 14, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-18-54", "timestamp": 1570767534, "time_this_iter_s": 289.7724847793579, "time_total_s": 3827.579626083374, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 3827.579626083374, "timesteps_since_restore": 840000, "iterations_since_restore": 14, "perf": {"cpu_util_percent": 62.149033816425124, "ram_util_percent": 50.398792270531395}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 1081.353797705279, "episode_reward_min": -2141.3476536829326, "episode_reward_mean": -446.89969052887267, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.10383208302859, "mean_processing_ms": 1.676796884919757, "mean_inference_ms": 0.9960639894286928}, "off_policy_estimator": {}, "info": {"num_steps_trained": 898560, "num_steps_sampled": 900000, "sample_time_ms": 289019.073, "load_time_ms": 13.447, "grad_time_ms": 7680.257, "update_time_ms": 5.651, "learner": {"default_policy": {"cur_kl_coeff": 0.20000000298023224, "cur_lr": 4.999999873689376e-05, "total_loss": 41.71358871459961, "policy_loss": -0.008507564663887024, "vf_loss": 41.720245361328125, "vf_explained_var": -2.292486422916795e-09, "kl": 0.009255404584109783, "entropy": 0.6216187477111816, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 900000, "episodes_total": 300, "training_iteration": 15, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-23-40", "timestamp": 1570767820, "time_this_iter_s": 285.5707211494446, "time_total_s": 4113.150347232819, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 4113.150347232819, "timesteps_since_restore": 900000, "iterations_since_restore": 15, "perf": {"cpu_util_percent": 61.191891891891885, "ram_util_percent": 50.94766584766585}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 1101.8440176363677, "episode_reward_min": -2141.3476536829326, "episode_reward_mean": -146.5685228869649, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.131669891983649, "mean_processing_ms": 1.6804058860614495, "mean_inference_ms": 1.00061880971467}, "off_policy_estimator": {}, "info": {"num_steps_trained": 958464, "num_steps_sampled": 960000, "sample_time_ms": 291389.424, "load_time_ms": 13.496, "grad_time_ms": 7744.557, "update_time_ms": 5.788, "learner": {"default_policy": {"cur_kl_coeff": 0.10000000149011612, "cur_lr": 4.999999873689376e-05, "total_loss": 46.670047760009766, "policy_loss": -0.008123466745018959, "vf_loss": 46.67705535888672, "vf_explained_var": -5.221774568298088e-09, "kl": 0.011164167895913124, "entropy": 0.5574238896369934, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 960000, "episodes_total": 320, "training_iteration": 16, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-28-27", "timestamp": 1570768107, "time_this_iter_s": 287.00454568862915, "time_total_s": 4400.154892921448, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 4400.154892921448, "timesteps_since_restore": 960000, "iterations_since_restore": 16, "perf": {"cpu_util_percent": 60.963080684596584, "ram_util_percent": 51.38117359413202}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 1515.6486623734095, "episode_reward_min": -1638.0720599202787, "episode_reward_mean": 129.62546895883378, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.156121243797343, "mean_processing_ms": 1.683453019331702, "mean_inference_ms": 1.004437954877005}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1018368, "num_steps_sampled": 1020000, "sample_time_ms": 290834.267, "load_time_ms": 13.311, "grad_time_ms": 7501.34, "update_time_ms": 5.64, "learner": {"default_policy": {"cur_kl_coeff": 0.10000000149011612, "cur_lr": 4.999999873689376e-05, "total_loss": 53.124549865722656, "policy_loss": -0.0074589853174984455, "vf_loss": 53.13093566894531, "vf_explained_var": -2.1651260784238957e-09, "kl": 0.010762547142803669, "entropy": 0.5129155516624451, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1020000, "episodes_total": 340, "training_iteration": 17, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-33-14", "timestamp": 1570768394, "time_this_iter_s": 286.7824501991272, "time_total_s": 4686.937343120575, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 4686.937343120575, "timesteps_since_restore": 1020000, "iterations_since_restore": 17, "perf": {"cpu_util_percent": 61.199999999999996, "ram_util_percent": 49.88068459657701}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 1660.846489412573, "episode_reward_min": -1034.344492390856, "episode_reward_mean": 361.5233325313951, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.181265086724607, "mean_processing_ms": 1.686685512821029, "mean_inference_ms": 1.008347753943032}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1078272, "num_steps_sampled": 1080000, "sample_time_ms": 286661.587, "load_time_ms": 13.458, "grad_time_ms": 7448.908, "update_time_ms": 5.653, "learner": {"default_policy": {"cur_kl_coeff": 0.10000000149011612, "cur_lr": 4.999999873689376e-05, "total_loss": 66.31993103027344, "policy_loss": -0.006132551934570074, "vf_loss": 66.32515716552734, "vf_explained_var": -8.787864658188482e-09, "kl": 0.009022381156682968, "entropy": 0.4667024612426758, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1080000, "episodes_total": 360, "training_iteration": 18, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-38-10", "timestamp": 1570768690, "time_this_iter_s": 296.4012680053711, "time_total_s": 4983.338611125946, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 4983.338611125946, "timesteps_since_restore": 1080000, "iterations_since_restore": 18, "perf": {"cpu_util_percent": 63.65886524822695, "ram_util_percent": 49.63569739952718}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 1744.5129197469237, "episode_reward_min": -680.3310403663952, "episode_reward_mean": 543.0221892787458, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.2068261160707685, "mean_processing_ms": 1.689920375901492, "mean_inference_ms": 1.0126095765594965}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1138176, "num_steps_sampled": 1140000, "sample_time_ms": 281797.399, "load_time_ms": 13.429, "grad_time_ms": 7264.563, "update_time_ms": 5.68, "learner": {"default_policy": {"cur_kl_coeff": 0.05000000074505806, "cur_lr": 4.999999873689376e-05, "total_loss": 91.78323364257812, "policy_loss": -0.004493150394409895, "vf_loss": 91.78736114501953, "vf_explained_var": 3.1840088343670914e-09, "kl": 0.007295538205653429, "entropy": 0.42863214015960693, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1140000, "episodes_total": 380, "training_iteration": 19, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-43-13", "timestamp": 1570768993, "time_this_iter_s": 303.31166338920593, "time_total_s": 5286.650274515152, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 5286.650274515152, "timesteps_since_restore": 1140000, "iterations_since_restore": 19, "perf": {"cpu_util_percent": 63.960046189376435, "ram_util_percent": 50.69214780600462}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 1857.1969157003678, "episode_reward_min": -533.1407552633883, "episode_reward_mean": 760.7714926088873, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.234304177337799, "mean_processing_ms": 1.6933028682206401, "mean_inference_ms": 1.0172869348789584}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1198080, "num_steps_sampled": 1200000, "sample_time_ms": 283712.481, "load_time_ms": 13.581, "grad_time_ms": 7323.056, "update_time_ms": 6.039, "learner": {"default_policy": {"cur_kl_coeff": 0.02500000037252903, "cur_lr": 4.999999873689376e-05, "total_loss": 132.9980010986328, "policy_loss": -0.003935374319553375, "vf_loss": 133.0017547607422, "vf_explained_var": -1.0188828669654981e-09, "kl": 0.007829303853213787, "entropy": 0.3700675666332245, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1200000, "episodes_total": 400, "training_iteration": 20, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-48-17", "timestamp": 1570769297, "time_this_iter_s": 303.47370505332947, "time_total_s": 5590.123979568481, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 5590.123979568481, "timesteps_since_restore": 1200000, "iterations_since_restore": 20, "perf": {"cpu_util_percent": 63.80925925925926, "ram_util_percent": 51.37939814814815}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 1885.0495048915807, "episode_reward_min": -533.1407552633883, "episode_reward_mean": 883.5141535219582, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.249043481372505, "mean_processing_ms": 1.6943665847044433, "mean_inference_ms": 1.019800933823224}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1257984, "num_steps_sampled": 1260000, "sample_time_ms": 279290.455, "load_time_ms": 13.348, "grad_time_ms": 7111.788, "update_time_ms": 6.362, "learner": {"default_policy": {"cur_kl_coeff": 0.012500000186264515, "cur_lr": 4.999999873689376e-05, "total_loss": 130.7291717529297, "policy_loss": -0.00409942539408803, "vf_loss": 130.733154296875, "vf_explained_var": -2.801927800888393e-09, "kl": 0.00828665029257536, "entropy": 0.3369026482105255, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1260000, "episodes_total": 420, "training_iteration": 21, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-52-18", "timestamp": 1570769538, "time_this_iter_s": 241.17777752876282, "time_total_s": 5831.301757097244, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 5831.301757097244, "timesteps_since_restore": 1260000, "iterations_since_restore": 21, "perf": {"cpu_util_percent": 54.82470930232558, "ram_util_percent": 41.23808139534884}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2018.8200815290775, "episode_reward_min": -106.56830443114728, "episode_reward_mean": 1018.4786740859113, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.25072077536659, "mean_processing_ms": 1.6932793552776686, "mean_inference_ms": 1.01977435153495}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1317888, "num_steps_sampled": 1320000, "sample_time_ms": 274102.656, "load_time_ms": 12.896, "grad_time_ms": 6912.674, "update_time_ms": 6.229, "learner": {"default_policy": {"cur_kl_coeff": 0.0062500000931322575, "cur_lr": 4.999999873689376e-05, "total_loss": 144.9283905029297, "policy_loss": -0.0042753373272717, "vf_loss": 144.93260192871094, "vf_explained_var": -3.1840088343670914e-09, "kl": 0.008732318878173828, "entropy": 0.2904764413833618, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1320000, "episodes_total": 440, "training_iteration": 22, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-56-10", "timestamp": 1570769770, "time_this_iter_s": 231.86295461654663, "time_total_s": 6063.164711713791, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 6063.164711713791, "timesteps_since_restore": 1320000, "iterations_since_restore": 22, "perf": {"cpu_util_percent": 55.576132930513594, "ram_util_percent": 38.05015105740182}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2018.8200815290775, "episode_reward_min": -106.56830443114728, "episode_reward_mean": 1121.6036093403736, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.236836122936435, "mean_processing_ms": 1.689467396021979, "mean_inference_ms": 1.0167814796001757}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1377792, "num_steps_sampled": 1380000, "sample_time_ms": 268413.283, "load_time_ms": 12.698, "grad_time_ms": 6748.455, "update_time_ms": 6.306, "learner": {"default_policy": {"cur_kl_coeff": 0.0031250000465661287, "cur_lr": 4.999999873689376e-05, "total_loss": 143.2877197265625, "policy_loss": -0.003045490710064769, "vf_loss": 143.29074096679688, "vf_explained_var": -8.405783624709784e-09, "kl": 0.006720947567373514, "entropy": 0.2587600648403168, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1380000, "episodes_total": 460, "training_iteration": 23, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_21-59-57", "timestamp": 1570769997, "time_this_iter_s": 226.61901426315308, "time_total_s": 6289.783725976944, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 6289.783725976944, "timesteps_since_restore": 1380000, "iterations_since_restore": 23, "perf": {"cpu_util_percent": 51.745061728395065, "ram_util_percent": 37.803395061728395}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2171.2679848606563, "episode_reward_min": -103.90599867112795, "episode_reward_mean": 1208.9556858688563, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.207254563235238, "mean_processing_ms": 1.6824385542798972, "mean_inference_ms": 1.0105801029295105}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1437696, "num_steps_sampled": 1440000, "sample_time_ms": 262176.436, "load_time_ms": 12.489, "grad_time_ms": 6535.291, "update_time_ms": 6.315, "learner": {"default_policy": {"cur_kl_coeff": 0.0015625000232830644, "cur_lr": 4.999999873689376e-05, "total_loss": 157.0971221923828, "policy_loss": -0.003159629413858056, "vf_loss": 157.1002655029297, "vf_explained_var": 2.4198467674096946e-09, "kl": 0.007727622054517269, "entropy": 0.22158615291118622, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1440000, "episodes_total": 480, "training_iteration": 24, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-03-42", "timestamp": 1570770222, "time_this_iter_s": 225.26621341705322, "time_total_s": 6515.049939393997, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 6515.049939393997, "timesteps_since_restore": 1440000, "iterations_since_restore": 24, "perf": {"cpu_util_percent": 51.45015576323988, "ram_util_percent": 37.77476635514018}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2177.282689812078, "episode_reward_min": -103.90599867112795, "episode_reward_mean": 1241.8399876042317, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.1654521680222185, "mean_processing_ms": 1.6733859633169788, "mean_inference_ms": 1.001977321894367}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1497600, "num_steps_sampled": 1500000, "sample_time_ms": 257629.305, "load_time_ms": 12.482, "grad_time_ms": 6411.156, "update_time_ms": 6.104, "learner": {"default_policy": {"cur_kl_coeff": 0.0007812500116415322, "cur_lr": 4.999999873689376e-05, "total_loss": 157.69520568847656, "policy_loss": -0.0025367350317537785, "vf_loss": 157.69775390625, "vf_explained_var": -1.2099233615003868e-08, "kl": 0.00578154856339097, "entropy": 0.18547388911247253, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1500000, "episodes_total": 500, "training_iteration": 25, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-07-41", "timestamp": 1570770461, "time_this_iter_s": 238.85392904281616, "time_total_s": 6753.903868436813, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 6753.903868436813, "timesteps_since_restore": 1500000, "iterations_since_restore": 25, "perf": {"cpu_util_percent": 55.407624633431084, "ram_util_percent": 40.25307917888563}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2346.6717922782086, "episode_reward_min": -35.48750666518595, "episode_reward_mean": 1336.7274591634734, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.124262021320062, "mean_processing_ms": 1.664400394419175, "mean_inference_ms": 0.9934427224858094}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1557504, "num_steps_sampled": 1560000, "sample_time_ms": 251716.412, "load_time_ms": 12.087, "grad_time_ms": 6207.78, "update_time_ms": 5.887, "learner": {"default_policy": {"cur_kl_coeff": 0.0003906250058207661, "cur_lr": 4.999999873689376e-05, "total_loss": 221.3697509765625, "policy_loss": -0.0030829834286123514, "vf_loss": 221.37283325195312, "vf_explained_var": -1.273603555951297e-09, "kl": 0.006834876723587513, "entropy": 0.1253022998571396, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1560000, "episodes_total": 520, "training_iteration": 26, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-11-27", "timestamp": 1570770687, "time_this_iter_s": 225.8318202495575, "time_total_s": 6979.735688686371, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 6979.735688686371, "timesteps_since_restore": 1560000, "iterations_since_restore": 26, "perf": {"cpu_util_percent": 51.75993788819876, "ram_util_percent": 37.82329192546583}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2473.0046252112115, "episode_reward_min": -35.48750666518595, "episode_reward_mean": 1444.352684346824, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.086305165552326, "mean_processing_ms": 1.6557962568319342, "mean_inference_ms": 0.9855702026004194}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1617408, "num_steps_sampled": 1620000, "sample_time_ms": 246342.935, "load_time_ms": 11.78, "grad_time_ms": 5994.347, "update_time_ms": 6.009, "learner": {"default_policy": {"cur_kl_coeff": 0.00019531250291038305, "cur_lr": 4.999999873689376e-05, "total_loss": 271.76629638671875, "policy_loss": -0.001978727290406823, "vf_loss": 271.76824951171875, "vf_explained_var": -6.240657324241283e-09, "kl": 0.0051463027484714985, "entropy": 0.0797242596745491, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1620000, "episodes_total": 540, "training_iteration": 27, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-15-17", "timestamp": 1570770917, "time_this_iter_s": 230.90782046318054, "time_total_s": 7210.643509149551, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 7210.643509149551, "timesteps_since_restore": 1620000, "iterations_since_restore": 27, "perf": {"cpu_util_percent": 54.23343465045593, "ram_util_percent": 37.77355623100304}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2473.0046252112115, "episode_reward_min": -35.48750666518595, "episode_reward_mean": 1501.5721173817913, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.050953980575351, "mean_processing_ms": 1.647533473699927, "mean_inference_ms": 0.9782470348249817}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1677312, "num_steps_sampled": 1680000, "sample_time_ms": 239298.803, "load_time_ms": 11.091, "grad_time_ms": 5733.348, "update_time_ms": 5.918, "learner": {"default_policy": {"cur_kl_coeff": 9.765625145519152e-05, "cur_lr": 4.999999873689376e-05, "total_loss": 203.982177734375, "policy_loss": -0.0018949565710499883, "vf_loss": 203.98406982421875, "vf_explained_var": -5.858576290762585e-09, "kl": 0.005937520414590836, "entropy": 0.07262831926345825, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1680000, "episodes_total": 560, "training_iteration": 28, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-19-01", "timestamp": 1570771141, "time_this_iter_s": 223.33587646484375, "time_total_s": 7433.979385614395, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 7433.979385614395, "timesteps_since_restore": 1680000, "iterations_since_restore": 28, "perf": {"cpu_util_percent": 51.34420062695926, "ram_util_percent": 37.992163009404386}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2473.0046252112115, "episode_reward_min": 662.7724399754616, "episode_reward_mean": 1553.8884773009734, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 6.018071470675672, "mean_processing_ms": 1.6397858567581012, "mean_inference_ms": 0.9714509168392169}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1737216, "num_steps_sampled": 1740000, "sample_time_ms": 231535.327, "load_time_ms": 10.841, "grad_time_ms": 5468.859, "update_time_ms": 5.12, "learner": {"default_policy": {"cur_kl_coeff": 4.882812572759576e-05, "cur_lr": 4.999999873689376e-05, "total_loss": 199.8420867919922, "policy_loss": -0.0020953549537807703, "vf_loss": 199.8441619873047, "vf_explained_var": -9.679387069638778e-09, "kl": 0.005188964307308197, "entropy": 0.0404723696410656, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1740000, "episodes_total": 580, "training_iteration": 29, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-22-44", "timestamp": 1570771364, "time_this_iter_s": 223.01602435112, "time_total_s": 7656.995409965515, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 7656.995409965515, "timesteps_since_restore": 1740000, "iterations_since_restore": 29, "perf": {"cpu_util_percent": 51.29150943396227, "ram_util_percent": 37.99559748427673}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2573.934205801572, "episode_reward_min": 824.2240294089684, "episode_reward_mean": 1659.2793911728618, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.985879629528855, "mean_processing_ms": 1.6318744501938778, "mean_inference_ms": 0.9647372745949525}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1797120, "num_steps_sampled": 1800000, "sample_time_ms": 224231.576, "load_time_ms": 10.168, "grad_time_ms": 5214.341, "update_time_ms": 4.914, "learner": {"default_policy": {"cur_kl_coeff": 2.441406286379788e-05, "cur_lr": 4.999999873689376e-05, "total_loss": 296.41766357421875, "policy_loss": -0.0018888604827225208, "vf_loss": 296.4195251464844, "vf_explained_var": -3.222216804488198e-08, "kl": 0.0052628349512815475, "entropy": -0.018881741911172867, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1800000, "episodes_total": 600, "training_iteration": 30, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-26-32", "timestamp": 1570771592, "time_this_iter_s": 227.87344074249268, "time_total_s": 7884.868850708008, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 7884.868850708008, "timesteps_since_restore": 1800000, "iterations_since_restore": 30, "perf": {"cpu_util_percent": 51.66676923076923, "ram_util_percent": 39.36923076923076}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2616.9848104271323, "episode_reward_min": 824.2240294089684, "episode_reward_mean": 1717.053522849597, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.9560246038940825, "mean_processing_ms": 1.6245303057639364, "mean_inference_ms": 0.9585262139783197}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1857024, "num_steps_sampled": 1860000, "sample_time_ms": 222865.698, "load_time_ms": 10.157, "grad_time_ms": 5211.954, "update_time_ms": 4.464, "learner": {"default_policy": {"cur_kl_coeff": 1.220703143189894e-05, "cur_lr": 4.999999873689376e-05, "total_loss": 308.9244384765625, "policy_loss": -0.0007358700968325138, "vf_loss": 308.9251403808594, "vf_explained_var": -1.7957809461677243e-08, "kl": 0.003371907165274024, "entropy": -0.0426129549741745, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1860000, "episodes_total": 620, "training_iteration": 31, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-30-19", "timestamp": 1570771819, "time_this_iter_s": 227.49058842658997, "time_total_s": 8112.359439134598, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 8112.359439134598, "timesteps_since_restore": 1860000, "iterations_since_restore": 31, "perf": {"cpu_util_percent": 51.446461538461534, "ram_util_percent": 39.654769230769226}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2616.9848104271323, "episode_reward_min": 824.2240294089684, "episode_reward_mean": 1769.2830190951656, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.927406970468294, "mean_processing_ms": 1.6176372363879121, "mean_inference_ms": 0.9526021235933171}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1916928, "num_steps_sampled": 1920000, "sample_time_ms": 222508.282, "load_time_ms": 10.181, "grad_time_ms": 5213.161, "update_time_ms": 4.482, "learner": {"default_policy": {"cur_kl_coeff": 6.10351571594947e-06, "cur_lr": 4.999999873689376e-05, "total_loss": 350.0728454589844, "policy_loss": -0.0010255386587232351, "vf_loss": 350.0738525390625, "vf_explained_var": -1.630212587144797e-08, "kl": 0.0036517020780593157, "entropy": -0.08226603269577026, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1920000, "episodes_total": 640, "training_iteration": 32, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-34-07", "timestamp": 1570772047, "time_this_iter_s": 228.3018355369568, "time_total_s": 8340.661274671555, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 8340.661274671555, "timesteps_since_restore": 1920000, "iterations_since_restore": 32, "perf": {"cpu_util_percent": 51.368307692307695, "ram_util_percent": 39.601230769230774}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2616.9848104271323, "episode_reward_min": 988.5700450267395, "episode_reward_mean": 1848.0285179073728, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.901369999157236, "mean_processing_ms": 1.6115178438951623, "mean_inference_ms": 0.9471200859892064}, "off_policy_estimator": {}, "info": {"num_steps_trained": 1976832, "num_steps_sampled": 1980000, "sample_time_ms": 222653.758, "load_time_ms": 10.167, "grad_time_ms": 5173.896, "update_time_ms": 4.438, "learner": {"default_policy": {"cur_kl_coeff": 3.051757857974735e-06, "cur_lr": 4.999999873689376e-05, "total_loss": 301.1153259277344, "policy_loss": -0.0014899128582328558, "vf_loss": 301.1168212890625, "vf_explained_var": -1.897669221762044e-08, "kl": 0.005265341140329838, "entropy": -0.1068715825676918, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 1980000, "episodes_total": 660, "training_iteration": 33, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-37-55", "timestamp": 1570772275, "time_this_iter_s": 227.6798312664032, "time_total_s": 8568.341105937958, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 8568.341105937958, "timesteps_since_restore": 1980000, "iterations_since_restore": 33, "perf": {"cpu_util_percent": 51.74861538461539, "ram_util_percent": 39.59938461538462}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2693.5456715325713, "episode_reward_min": 988.5700450267395, "episode_reward_mean": 1911.9438265035753, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.877489217012647, "mean_processing_ms": 1.605949389614291, "mean_inference_ms": 0.9420209901324407}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2036736, "num_steps_sampled": 2040000, "sample_time_ms": 222723.351, "load_time_ms": 10.157, "grad_time_ms": 5170.939, "update_time_ms": 4.437, "learner": {"default_policy": {"cur_kl_coeff": 1.5258789289873675e-06, "cur_lr": 4.999999873689376e-05, "total_loss": 293.3316650390625, "policy_loss": -0.0008449004380963743, "vf_loss": 293.3324890136719, "vf_explained_var": -5.349134912790987e-09, "kl": 0.004025230649858713, "entropy": -0.11745946854352951, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2040000, "episodes_total": 680, "training_iteration": 34, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-41-41", "timestamp": 1570772501, "time_this_iter_s": 225.9329686164856, "time_total_s": 8794.274074554443, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 8794.274074554443, "timesteps_since_restore": 2040000, "iterations_since_restore": 34, "perf": {"cpu_util_percent": 51.452173913043474, "ram_util_percent": 39.59937888198758}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2693.5456715325713, "episode_reward_min": 988.5700450267395, "episode_reward_mean": 1936.2141707487754, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.854872266222085, "mean_processing_ms": 1.6008661423817119, "mean_inference_ms": 0.9372015370668023}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2096640, "num_steps_sampled": 2100000, "sample_time_ms": 221565.32, "load_time_ms": 10.107, "grad_time_ms": 5096.784, "update_time_ms": 4.341, "learner": {"default_policy": {"cur_kl_coeff": 7.629394644936838e-07, "cur_lr": 4.999999873689376e-05, "total_loss": 320.0366516113281, "policy_loss": -0.0009304272243753076, "vf_loss": 320.0376281738281, "vf_explained_var": 1.528324244937096e-09, "kl": 0.0032853041775524616, "entropy": -0.15430699288845062, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2100000, "episodes_total": 700, "training_iteration": 35, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-45-28", "timestamp": 1570772728, "time_this_iter_s": 226.53039646148682, "time_total_s": 9020.80447101593, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 9020.80447101593, "timesteps_since_restore": 2100000, "iterations_since_restore": 35, "perf": {"cpu_util_percent": 51.77368421052632, "ram_util_percent": 39.60061919504644}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2693.5456715325713, "episode_reward_min": 988.5700450267395, "episode_reward_mean": 1930.5465885275719, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.833342924196859, "mean_processing_ms": 1.5960547506565224, "mean_inference_ms": 0.9326516376453324}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2156544, "num_steps_sampled": 2160000, "sample_time_ms": 221577.781, "load_time_ms": 10.131, "grad_time_ms": 5104.264, "update_time_ms": 4.325, "learner": {"default_policy": {"cur_kl_coeff": 3.814697322468419e-07, "cur_lr": 4.999999873689376e-05, "total_loss": 292.1423034667969, "policy_loss": -0.0013726623728871346, "vf_loss": 292.1437072753906, "vf_explained_var": -2.2160701718121345e-08, "kl": 0.004224446602165699, "entropy": -0.16808035969734192, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2160000, "episodes_total": 720, "training_iteration": 36, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-49-14", "timestamp": 1570772954, "time_this_iter_s": 226.03132367134094, "time_total_s": 9246.835794687271, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 9246.835794687271, "timesteps_since_restore": 2160000, "iterations_since_restore": 36, "perf": {"cpu_util_percent": 51.382972136222925, "ram_util_percent": 39.59969040247679}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2776.6405679157083, "episode_reward_min": 1093.7900345984647, "episode_reward_mean": 1945.0163757393439, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.812983189318344, "mean_processing_ms": 1.591521016067461, "mean_inference_ms": 0.9283693052454841}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2216448, "num_steps_sampled": 2220000, "sample_time_ms": 221296.094, "load_time_ms": 10.097, "grad_time_ms": 5115.562, "update_time_ms": 4.328, "learner": {"default_policy": {"cur_kl_coeff": 1.9073486612342094e-07, "cur_lr": 4.999999873689376e-05, "total_loss": 367.0290222167969, "policy_loss": -0.0010790476808324456, "vf_loss": 367.03009033203125, "vf_explained_var": -2.6363592198208607e-08, "kl": 0.006083440966904163, "entropy": -0.20631200075149536, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2220000, "episodes_total": 740, "training_iteration": 37, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-53-02", "timestamp": 1570773182, "time_this_iter_s": 228.20357489585876, "time_total_s": 9475.03936958313, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 9475.03936958313, "timesteps_since_restore": 2220000, "iterations_since_restore": 37, "perf": {"cpu_util_percent": 51.33046153846154, "ram_util_percent": 39.60092307692309}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2776.6405679157083, "episode_reward_min": 1156.7972621860602, "episode_reward_mean": 1950.0848381205637, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.793628959826196, "mean_processing_ms": 1.5871995447056495, "mean_inference_ms": 0.9243322535528041}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2276352, "num_steps_sampled": 2280000, "sample_time_ms": 221657.953, "load_time_ms": 10.129, "grad_time_ms": 5116.435, "update_time_ms": 4.355, "learner": {"default_policy": {"cur_kl_coeff": 9.536743306171047e-08, "cur_lr": 4.999999873689376e-05, "total_loss": 312.8647155761719, "policy_loss": -0.0005480647087097168, "vf_loss": 312.86529541015625, "vf_explained_var": -4.610444648278644e-08, "kl": 0.0033533277455717325, "entropy": -0.21347717940807343, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2280000, "episodes_total": 760, "training_iteration": 38, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_22-56-49", "timestamp": 1570773409, "time_this_iter_s": 226.96341514587402, "time_total_s": 9702.002784729004, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 9702.002784729004, "timesteps_since_restore": 2280000, "iterations_since_restore": 38, "perf": {"cpu_util_percent": 51.65370370370369, "ram_util_percent": 39.6}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2776.6405679157083, "episode_reward_min": 1037.897900478011, "episode_reward_mean": 1962.2869825974358, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.775375571550589, "mean_processing_ms": 1.5831787139466353, "mean_inference_ms": 0.9205203030347175}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2336256, "num_steps_sampled": 2340000, "sample_time_ms": 221979.38, "load_time_ms": 10.173, "grad_time_ms": 5122.914, "update_time_ms": 4.401, "learner": {"default_policy": {"cur_kl_coeff": 4.7683716530855236e-08, "cur_lr": 4.999999873689376e-05, "total_loss": 312.23712158203125, "policy_loss": -0.0005565821775235236, "vf_loss": 312.2376403808594, "vf_explained_var": -4.1137393935741784e-08, "kl": 0.0023237953428179026, "entropy": -0.22383910417556763, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2340000, "episodes_total": 780, "training_iteration": 39, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-00-35", "timestamp": 1570773635, "time_this_iter_s": 226.29635906219482, "time_total_s": 9928.299143791199, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 9928.299143791199, "timesteps_since_restore": 2340000, "iterations_since_restore": 39, "perf": {"cpu_util_percent": 51.54272445820433, "ram_util_percent": 39.60092879256967}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2776.6405679157083, "episode_reward_min": 1037.897900478011, "episode_reward_mean": 1974.5461355925602, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.758149660119385, "mean_processing_ms": 1.5793229004773917, "mean_inference_ms": 0.9169234915072964}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2396160, "num_steps_sampled": 2400000, "sample_time_ms": 222022.82, "load_time_ms": 10.228, "grad_time_ms": 5110.49, "update_time_ms": 4.407, "learner": {"default_policy": {"cur_kl_coeff": 2.3841858265427618e-08, "cur_lr": 4.999999873689376e-05, "total_loss": 338.8609313964844, "policy_loss": -0.0010798104340210557, "vf_loss": 338.86199951171875, "vf_explained_var": -3.718922414464032e-08, "kl": 0.003974047023802996, "entropy": -0.26470670104026794, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2400000, "episodes_total": 800, "training_iteration": 40, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-04-23", "timestamp": 1570773863, "time_this_iter_s": 228.18473315238953, "time_total_s": 10156.483876943588, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 10156.483876943588, "timesteps_since_restore": 2400000, "iterations_since_restore": 40, "perf": {"cpu_util_percent": 51.333742331288356, "ram_util_percent": 39.612576687116565}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2784.3382585599466, "episode_reward_min": 1037.897900478011, "episode_reward_mean": 2034.1331183414325, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.742247119261552, "mean_processing_ms": 1.575586036721535, "mean_inference_ms": 0.913545587494423}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2456064, "num_steps_sampled": 2460000, "sample_time_ms": 222069.957, "load_time_ms": 10.194, "grad_time_ms": 5114.018, "update_time_ms": 4.496, "learner": {"default_policy": {"cur_kl_coeff": 1.1920929132713809e-08, "cur_lr": 4.999999873689376e-05, "total_loss": 372.523193359375, "policy_loss": -0.00048392629832960665, "vf_loss": 372.5236511230469, "vf_explained_var": -3.324105080082518e-08, "kl": 0.003001442179083824, "entropy": -0.2981806993484497, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2460000, "episodes_total": 820, "training_iteration": 41, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-08-11", "timestamp": 1570774091, "time_this_iter_s": 227.9981415271759, "time_total_s": 10384.482018470764, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 10384.482018470764, "timesteps_since_restore": 2460000, "iterations_since_restore": 41, "perf": {"cpu_util_percent": 51.42553846153847, "ram_util_percent": 39.63876923076923}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2784.3382585599466, "episode_reward_min": 909.305565800551, "episode_reward_mean": 1978.7946372907566, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.7267659215024, "mean_processing_ms": 1.5718495197045461, "mean_inference_ms": 0.9103305837742315}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2515968, "num_steps_sampled": 2520000, "sample_time_ms": 221636.299, "load_time_ms": 10.175, "grad_time_ms": 5101.992, "update_time_ms": 4.505, "learner": {"default_policy": {"cur_kl_coeff": 5.9604645663569045e-09, "cur_lr": 4.999999873689376e-05, "total_loss": 276.86187744140625, "policy_loss": -0.0002941251150332391, "vf_loss": 276.8622131347656, "vf_explained_var": -3.094856637630983e-08, "kl": 0.003100535599514842, "entropy": -0.2838324010372162, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2520000, "episodes_total": 840, "training_iteration": 42, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-11-55", "timestamp": 1570774315, "time_this_iter_s": 223.84502983093262, "time_total_s": 10608.327048301697, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 10608.327048301697, "timesteps_since_restore": 2520000, "iterations_since_restore": 42, "perf": {"cpu_util_percent": 51.5, "ram_util_percent": 39.690282131661434}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2784.3382585599466, "episode_reward_min": 909.305565800551, "episode_reward_mean": 2004.5044007690408, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.712289092222843, "mean_processing_ms": 1.5683120312660341, "mean_inference_ms": 0.9073048101891489}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2575872, "num_steps_sampled": 2580000, "sample_time_ms": 221853.994, "load_time_ms": 10.18, "grad_time_ms": 5100.811, "update_time_ms": 4.514, "learner": {"default_policy": {"cur_kl_coeff": 2.9802322831784522e-09, "cur_lr": 4.999999873689376e-05, "total_loss": 342.7833557128906, "policy_loss": -0.0010411189869046211, "vf_loss": 342.784423828125, "vf_explained_var": -2.610887150922281e-08, "kl": 0.006138394586741924, "entropy": -0.32215142250061035, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2580000, "episodes_total": 860, "training_iteration": 43, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-15-45", "timestamp": 1570774545, "time_this_iter_s": 229.84651684761047, "time_total_s": 10838.173565149307, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 10838.173565149307, "timesteps_since_restore": 2580000, "iterations_since_restore": 43, "perf": {"cpu_util_percent": 53.3734756097561, "ram_util_percent": 39.767682926829266}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2784.3382585599466, "episode_reward_min": 909.305565800551, "episode_reward_mean": 2001.5831379296658, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.698582902120918, "mean_processing_ms": 1.564866482511378, "mean_inference_ms": 0.9044546734094819}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2635776, "num_steps_sampled": 2640000, "sample_time_ms": 221920.955, "load_time_ms": 10.225, "grad_time_ms": 5099.336, "update_time_ms": 4.514, "learner": {"default_policy": {"cur_kl_coeff": 1.4901161415892261e-09, "cur_lr": 4.999999873689376e-05, "total_loss": 296.57952880859375, "policy_loss": -0.0004189646278973669, "vf_loss": 296.5799560546875, "vf_explained_var": 2.6745674563954935e-09, "kl": 0.0030859194230288267, "entropy": -0.31912147998809814, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2640000, "episodes_total": 880, "training_iteration": 44, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-19-32", "timestamp": 1570774772, "time_this_iter_s": 226.58823561668396, "time_total_s": 11064.761800765991, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 11064.761800765991, "timesteps_since_restore": 2640000, "iterations_since_restore": 44, "perf": {"cpu_util_percent": 51.426934984520116, "ram_util_percent": 39.89938080495355}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2784.3382585599466, "episode_reward_min": 909.305565800551, "episode_reward_mean": 1958.6506716824038, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.685303008469853, "mean_processing_ms": 1.5616100196434957, "mean_inference_ms": 0.9017471943642369}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2695680, "num_steps_sampled": 2700000, "sample_time_ms": 221985.268, "load_time_ms": 10.197, "grad_time_ms": 5098.447, "update_time_ms": 4.628, "learner": {"default_policy": {"cur_kl_coeff": 7.450580707946131e-10, "cur_lr": 4.999999873689376e-05, "total_loss": 265.79290771484375, "policy_loss": -0.0009432857623323798, "vf_loss": 265.7938232421875, "vf_explained_var": -5.094414223805188e-09, "kl": 0.002575733233243227, "entropy": -0.3287334144115448, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2700000, "episodes_total": 900, "training_iteration": 45, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-23-19", "timestamp": 1570774999, "time_this_iter_s": 227.1653265953064, "time_total_s": 11291.927127361298, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 11291.927127361298, "timesteps_since_restore": 2700000, "iterations_since_restore": 45, "perf": {"cpu_util_percent": 51.33302469135802, "ram_util_percent": 39.89907407407407}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2764.5460546960426, "episode_reward_min": 909.305565800551, "episode_reward_mean": 1940.7206650363337, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.67261003747345, "mean_processing_ms": 1.5586849210772256, "mean_inference_ms": 0.899194261659875}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2755584, "num_steps_sampled": 2760000, "sample_time_ms": 222211.219, "load_time_ms": 10.21, "grad_time_ms": 5100.666, "update_time_ms": 4.646, "learner": {"default_policy": {"cur_kl_coeff": 3.7252903539730653e-10, "cur_lr": 4.999999873689376e-05, "total_loss": 340.6348571777344, "policy_loss": -0.00033470403286628425, "vf_loss": 340.63519287109375, "vf_explained_var": -1.7830449339228949e-09, "kl": 0.003041619434952736, "entropy": -0.3805379569530487, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2760000, "episodes_total": 920, "training_iteration": 46, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-27-07", "timestamp": 1570775227, "time_this_iter_s": 228.3132302761078, "time_total_s": 11520.240357637405, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 11520.240357637405, "timesteps_since_restore": 2760000, "iterations_since_restore": 46, "perf": {"cpu_util_percent": 51.72883435582822, "ram_util_percent": 39.899386503067475}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2764.5460546960426, "episode_reward_min": 1118.3477549045203, "episode_reward_mean": 2003.4134713914696, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.6610115785514825, "mean_processing_ms": 1.5559140257059776, "mean_inference_ms": 0.8968104054655179}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2815488, "num_steps_sampled": 2820000, "sample_time_ms": 222113.488, "load_time_ms": 10.191, "grad_time_ms": 5098.286, "update_time_ms": 4.636, "learner": {"default_policy": {"cur_kl_coeff": 1.8626451769865326e-10, "cur_lr": 4.999999873689376e-05, "total_loss": 369.5340576171875, "policy_loss": -3.198018475814024e-06, "vf_loss": 369.5340576171875, "vf_explained_var": -2.1651260784238957e-09, "kl": 0.003405811032280326, "entropy": -0.3966348469257355, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2820000, "episodes_total": 940, "training_iteration": 47, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-30-54", "timestamp": 1570775454, "time_this_iter_s": 227.20234751701355, "time_total_s": 11747.442705154419, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 11747.442705154419, "timesteps_since_restore": 2820000, "iterations_since_restore": 47, "perf": {"cpu_util_percent": 51.59814814814815, "ram_util_percent": 39.89938271604938}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2763.541614495298, "episode_reward_min": 1118.3477549045203, "episode_reward_mean": 1935.2097842175808, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.649386125975444, "mean_processing_ms": 1.553293031933046, "mean_inference_ms": 0.8945072540306853}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2875392, "num_steps_sampled": 2880000, "sample_time_ms": 221942.58, "load_time_ms": 10.14, "grad_time_ms": 5100.424, "update_time_ms": 4.62, "learner": {"default_policy": {"cur_kl_coeff": 9.313225884932663e-11, "cur_lr": 4.999999873689376e-05, "total_loss": 235.3415069580078, "policy_loss": -0.0004265991155989468, "vf_loss": 235.34194946289062, "vf_explained_var": -6.622738357719982e-09, "kl": 0.003912602551281452, "entropy": -0.3681495189666748, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2880000, "episodes_total": 960, "training_iteration": 48, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-34-40", "timestamp": 1570775680, "time_this_iter_s": 225.27441596984863, "time_total_s": 11972.717121124268, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 11972.717121124268, "timesteps_since_restore": 2880000, "iterations_since_restore": 48, "perf": {"cpu_util_percent": 51.63447204968945, "ram_util_percent": 39.9}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2763.541614495298, "episode_reward_min": 1118.3477549045203, "episode_reward_mean": 1952.9976265574605, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.638318252048253, "mean_processing_ms": 1.5508499834906309, "mean_inference_ms": 0.8922921938687176}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2935296, "num_steps_sampled": 2940000, "sample_time_ms": 222022.731, "load_time_ms": 10.124, "grad_time_ms": 5096.844, "update_time_ms": 4.629, "learner": {"default_policy": {"cur_kl_coeff": 4.6566129424663316e-11, "cur_lr": 4.999999873689376e-05, "total_loss": 314.6124267578125, "policy_loss": -0.00013308048073668033, "vf_loss": 314.6125183105469, "vf_explained_var": -4.839693534819389e-09, "kl": 0.003223870415240526, "entropy": -0.40551745891571045, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 2940000, "episodes_total": 980, "training_iteration": 49, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-38-27", "timestamp": 1570775907, "time_this_iter_s": 227.061918258667, "time_total_s": 12199.779039382935, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 12199.779039382935, "timesteps_since_restore": 2940000, "iterations_since_restore": 49, "perf": {"cpu_util_percent": 51.55572755417957, "ram_util_percent": 39.89969040247678}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2763.541614495298, "episode_reward_min": 1159.3331360048546, "episode_reward_mean": 1989.8064041701018, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.628015489307915, "mean_processing_ms": 1.5484684826664992, "mean_inference_ms": 0.8902062657092839}, "off_policy_estimator": {}, "info": {"num_steps_trained": 2995200, "num_steps_sampled": 3000000, "sample_time_ms": 221989.03, "load_time_ms": 10.1, "grad_time_ms": 5100.174, "update_time_ms": 4.647, "learner": {"default_policy": {"cur_kl_coeff": 2.3283064712331658e-11, "cur_lr": 4.999999873689376e-05, "total_loss": 319.1338806152344, "policy_loss": -0.0006387312314473093, "vf_loss": 319.1344909667969, "vf_explained_var": -4.0755314678619925e-09, "kl": 0.002699590288102627, "entropy": -0.42596644163131714, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3000000, "episodes_total": 1000, "training_iteration": 50, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-42-15", "timestamp": 1570776135, "time_this_iter_s": 227.88091659545898, "time_total_s": 12427.659955978394, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 12427.659955978394, "timesteps_since_restore": 3000000, "iterations_since_restore": 50, "perf": {"cpu_util_percent": 51.739570552147235, "ram_util_percent": 39.90153374233128}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2747.2897032528185, "episode_reward_min": 1142.1583378230064, "episode_reward_mean": 1926.810640347007, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.61774479317638, "mean_processing_ms": 1.5460156789176551, "mean_inference_ms": 0.8881574764052952}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3055104, "num_steps_sampled": 3060000, "sample_time_ms": 221677.828, "load_time_ms": 10.137, "grad_time_ms": 5095.029, "update_time_ms": 4.618, "learner": {"default_policy": {"cur_kl_coeff": 1.1641532356165829e-11, "cur_lr": 4.999999873689376e-05, "total_loss": 244.47679138183594, "policy_loss": -0.0007065013633109629, "vf_loss": 244.47750854492188, "vf_explained_var": -2.1651260784238957e-09, "kl": 0.0038327837828546762, "entropy": -0.4098336696624756, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3060000, "episodes_total": 1020, "training_iteration": 51, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-46-00", "timestamp": 1570776360, "time_this_iter_s": 224.83476877212524, "time_total_s": 12652.494724750519, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 12652.494724750519, "timesteps_since_restore": 3060000, "iterations_since_restore": 51, "perf": {"cpu_util_percent": 51.411874999999995, "ram_util_percent": 39.89999999999999}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2747.2897032528185, "episode_reward_min": 849.5323429650005, "episode_reward_mean": 1846.3953676391588, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.607525936256994, "mean_processing_ms": 1.5436799964025325, "mean_inference_ms": 0.8861643324832422}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3115008, "num_steps_sampled": 3120000, "sample_time_ms": 221741.947, "load_time_ms": 10.18, "grad_time_ms": 5090.579, "update_time_ms": 4.619, "learner": {"default_policy": {"cur_kl_coeff": 5.8207661780829145e-12, "cur_lr": 4.999999873689376e-05, "total_loss": 246.7576904296875, "policy_loss": -0.000154356675921008, "vf_loss": 246.75784301757812, "vf_explained_var": -1.1080350859060673e-08, "kl": 0.0025222529657185078, "entropy": -0.42039740085601807, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3120000, "episodes_total": 1040, "training_iteration": 52, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-49-44", "timestamp": 1570776584, "time_this_iter_s": 224.4415910243988, "time_total_s": 12876.936315774918, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 12876.936315774918, "timesteps_since_restore": 3120000, "iterations_since_restore": 52, "perf": {"cpu_util_percent": 51.7865625, "ram_util_percent": 38.92}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2747.2897032528185, "episode_reward_min": 849.5323429650005, "episode_reward_mean": 1840.7385588912368, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.597553581817157, "mean_processing_ms": 1.541261210592515, "mean_inference_ms": 0.8842086547485476}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3174912, "num_steps_sampled": 3180000, "sample_time_ms": 221078.755, "load_time_ms": 10.153, "grad_time_ms": 5080.255, "update_time_ms": 4.664, "learner": {"default_policy": {"cur_kl_coeff": 2.9103830890414573e-12, "cur_lr": 4.999999873689376e-05, "total_loss": 231.25408935546875, "policy_loss": -0.0010070799617096782, "vf_loss": 231.25509643554688, "vf_explained_var": -3.8208107788761936e-09, "kl": 0.006194713991135359, "entropy": -0.42542222142219543, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3180000, "episodes_total": 1060, "training_iteration": 53, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-53-27", "timestamp": 1570776807, "time_this_iter_s": 223.1104326248169, "time_total_s": 13100.046748399734, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 13100.046748399734, "timesteps_since_restore": 3180000, "iterations_since_restore": 53, "perf": {"cpu_util_percent": 51.20501567398119, "ram_util_percent": 38.3037617554859}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2788.2348789393354, "episode_reward_min": 849.5323429650005, "episode_reward_mean": 1830.1222636539665, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.587707852952758, "mean_processing_ms": 1.5389584259441795, "mean_inference_ms": 0.882305901526306}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3234816, "num_steps_sampled": 3240000, "sample_time_ms": 220918.763, "load_time_ms": 10.157, "grad_time_ms": 5075.448, "update_time_ms": 4.723, "learner": {"default_policy": {"cur_kl_coeff": 1.4551915445207286e-12, "cur_lr": 4.999999873689376e-05, "total_loss": 304.02984619140625, "policy_loss": 7.040964555926621e-05, "vf_loss": 304.02978515625, "vf_explained_var": -5.7312159462696854e-09, "kl": 0.0009638385381549597, "entropy": -0.4713115394115448, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3240000, "episodes_total": 1080, "training_iteration": 54, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-10_23-57-12", "timestamp": 1570777032, "time_this_iter_s": 224.94078087806702, "time_total_s": 13324.987529277802, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 13324.987529277802, "timesteps_since_restore": 3240000, "iterations_since_restore": 54, "perf": {"cpu_util_percent": 51.45077881619938, "ram_util_percent": 38.304361370716514}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2788.2348789393354, "episode_reward_min": 849.5323429650005, "episode_reward_mean": 1785.6505683487608, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.577708271921396, "mean_processing_ms": 1.5366353519314884, "mean_inference_ms": 0.8804114250094345}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3294720, "num_steps_sampled": 3300000, "sample_time_ms": 220532.096, "load_time_ms": 10.185, "grad_time_ms": 5061.38, "update_time_ms": 4.633, "learner": {"default_policy": {"cur_kl_coeff": 7.275957722603643e-13, "cur_lr": 4.999999873689376e-05, "total_loss": 251.47271728515625, "policy_loss": -8.139714918797836e-05, "vf_loss": 251.4728240966797, "vf_explained_var": -2.0377657339309962e-09, "kl": 0.0022987022530287504, "entropy": -0.4628976583480835, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3300000, "episodes_total": 1100, "training_iteration": 55, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-00-55", "timestamp": 1570777255, "time_this_iter_s": 223.15795159339905, "time_total_s": 13548.1454808712, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 13548.1454808712, "timesteps_since_restore": 3300000, "iterations_since_restore": 55, "perf": {"cpu_util_percent": 51.31792452830188, "ram_util_percent": 38.30471698113208}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2788.2348789393354, "episode_reward_min": 849.5323429650005, "episode_reward_mean": 1822.2845389159845, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.568417337633138, "mean_processing_ms": 1.5345191609522713, "mean_inference_ms": 0.8786465458495388}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3354624, "num_steps_sampled": 3360000, "sample_time_ms": 220588.654, "load_time_ms": 10.199, "grad_time_ms": 5067.143, "update_time_ms": 4.632, "learner": {"default_policy": {"cur_kl_coeff": 3.6379788613018216e-13, "cur_lr": 4.999999873689376e-05, "total_loss": 293.00921630859375, "policy_loss": -0.0003284713311586529, "vf_loss": 293.0095520019531, "vf_explained_var": -2.0377657339309962e-09, "kl": 0.005179738160222769, "entropy": -0.49104171991348267, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3360000, "episodes_total": 1120, "training_iteration": 56, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-04-44", "timestamp": 1570777484, "time_this_iter_s": 228.93655133247375, "time_total_s": 13777.082032203674, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 13777.082032203674, "timesteps_since_restore": 3360000, "iterations_since_restore": 56, "perf": {"cpu_util_percent": 51.663914373088694, "ram_util_percent": 39.33058103975536}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2788.2348789393354, "episode_reward_min": 943.0457249174575, "episode_reward_mean": 1837.8479881899118, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.559658109902752, "mean_processing_ms": 1.5326212045498182, "mean_inference_ms": 0.8769971832419584}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3414528, "num_steps_sampled": 3420000, "sample_time_ms": 220734.391, "load_time_ms": 10.192, "grad_time_ms": 5064.238, "update_time_ms": 4.609, "learner": {"default_policy": {"cur_kl_coeff": 1.8189894306509108e-13, "cur_lr": 4.999999873689376e-05, "total_loss": 262.3144226074219, "policy_loss": -0.0002233254926977679, "vf_loss": 262.3146667480469, "vf_explained_var": -8.151062935723985e-09, "kl": 0.004653227515518665, "entropy": -0.48141250014305115, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3420000, "episodes_total": 1140, "training_iteration": 57, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-08-33", "timestamp": 1570777713, "time_this_iter_s": 228.6304793357849, "time_total_s": 14005.71251153946, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 14005.71251153946, "timesteps_since_restore": 3420000, "iterations_since_restore": 57, "perf": {"cpu_util_percent": 52.07607361963191, "ram_util_percent": 40.06993865030675}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2788.2348789393354, "episode_reward_min": 1013.4801842347422, "episode_reward_mean": 1851.014425770698, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.551316000690456, "mean_processing_ms": 1.5308382366995494, "mean_inference_ms": 0.8754184791119344}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3474432, "num_steps_sampled": 3480000, "sample_time_ms": 220617.594, "load_time_ms": 10.271, "grad_time_ms": 5058.143, "update_time_ms": 4.658, "learner": {"default_policy": {"cur_kl_coeff": 9.094947153254554e-14, "cur_lr": 4.999999873689376e-05, "total_loss": 250.1034393310547, "policy_loss": 0.00013227085582911968, "vf_loss": 250.10330200195312, "vf_explained_var": -8.023702591231086e-09, "kl": 0.0014566911850124598, "entropy": -0.4811890423297882, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3480000, "episodes_total": 1160, "training_iteration": 58, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-12-17", "timestamp": 1570777937, "time_this_iter_s": 224.04734444618225, "time_total_s": 14229.759855985641, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 14229.759855985641, "timesteps_since_restore": 3480000, "iterations_since_restore": 58, "perf": {"cpu_util_percent": 51.46551724137931, "ram_util_percent": 39.995611285266456}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2649.4182374819, "episode_reward_min": 863.929739874468, "episode_reward_mean": 1804.1985942052709, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.543606926669899, "mean_processing_ms": 1.529229142794223, "mean_inference_ms": 0.8739786483460893}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3534336, "num_steps_sampled": 3540000, "sample_time_ms": 220977.904, "load_time_ms": 10.284, "grad_time_ms": 5059.191, "update_time_ms": 4.602, "learner": {"default_policy": {"cur_kl_coeff": 4.547473576627277e-14, "cur_lr": 4.999999873689376e-05, "total_loss": 235.62156677246094, "policy_loss": 0.000111854009446688, "vf_loss": 235.62142944335938, "vf_explained_var": 3.82081061234274e-10, "kl": 0.00260219001211226, "entropy": -0.4805951714515686, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3540000, "episodes_total": 1180, "training_iteration": 59, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-16-08", "timestamp": 1570778168, "time_this_iter_s": 230.6747522354126, "time_total_s": 14460.434608221054, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 14460.434608221054, "timesteps_since_restore": 3540000, "iterations_since_restore": 59, "perf": {"cpu_util_percent": 54.111854103343475, "ram_util_percent": 40.10911854103343}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2635.2257878794317, "episode_reward_min": 863.929739874468, "episode_reward_mean": 1778.3939188339803, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.538648203625481, "mean_processing_ms": 1.5285080907098614, "mean_inference_ms": 0.8731121956970821}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3594240, "num_steps_sampled": 3600000, "sample_time_ms": 223877.151, "load_time_ms": 10.874, "grad_time_ms": 5401.208, "update_time_ms": 4.615, "learner": {"default_policy": {"cur_kl_coeff": 2.2737367883136385e-14, "cur_lr": 4.999999873689376e-05, "total_loss": 208.59107971191406, "policy_loss": -0.0010167751461267471, "vf_loss": 208.59210205078125, "vf_explained_var": -4.330252156847791e-09, "kl": 0.0033729688730090857, "entropy": -0.5039448142051697, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3600000, "episodes_total": 1200, "training_iteration": 60, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-20-28", "timestamp": 1570778428, "time_this_iter_s": 260.3004310131073, "time_total_s": 14720.735039234161, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 14720.735039234161, "timesteps_since_restore": 3600000, "iterations_since_restore": 60, "perf": {"cpu_util_percent": 60.22096774193549, "ram_util_percent": 46.51559139784947}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2745.0966124556626, "episode_reward_min": 863.929739874468, "episode_reward_mean": 1799.3220629325765, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.542103984451785, "mean_processing_ms": 1.5293023327910533, "mean_inference_ms": 0.8739815761190226}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3654144, "num_steps_sampled": 3660000, "sample_time_ms": 234727.62, "load_time_ms": 10.904, "grad_time_ms": 5390.841, "update_time_ms": 4.573, "learner": {"default_policy": {"cur_kl_coeff": 1.1368683941568192e-14, "cur_lr": 4.999999873689376e-05, "total_loss": 319.5943603515625, "policy_loss": -0.0012102772016078234, "vf_loss": 319.5955505371094, "vf_explained_var": 5.094414334827491e-10, "kl": 0.004356545861810446, "entropy": -0.5843683481216431, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3660000, "episodes_total": 1220, "training_iteration": 61, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-26-01", "timestamp": 1570778761, "time_this_iter_s": 333.2357165813446, "time_total_s": 15053.970755815506, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 15053.970755815506, "timesteps_since_restore": 3660000, "iterations_since_restore": 61, "perf": {"cpu_util_percent": 69.3861052631579, "ram_util_percent": 52.766315789473694}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2745.0966124556626, "episode_reward_min": 863.929739874468, "episode_reward_mean": 1842.7114199770353, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.545520620895262, "mean_processing_ms": 1.5299943937683245, "mean_inference_ms": 0.8748540436520134}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3714048, "num_steps_sampled": 3720000, "sample_time_ms": 234838.326, "load_time_ms": 10.896, "grad_time_ms": 5389.675, "update_time_ms": 4.496, "learner": {"default_policy": {"cur_kl_coeff": 5.684341970784096e-15, "cur_lr": 4.999999873689376e-05, "total_loss": 316.5411376953125, "policy_loss": -0.00012951325334142894, "vf_loss": 316.541259765625, "vf_explained_var": -6.495378013227082e-09, "kl": 0.004084589891135693, "entropy": -0.5971006751060486, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3720000, "episodes_total": 1240, "training_iteration": 62, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-29-47", "timestamp": 1570778987, "time_this_iter_s": 225.53657627105713, "time_total_s": 15279.507332086563, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 15279.507332086563, "timesteps_since_restore": 3720000, "iterations_since_restore": 62, "perf": {"cpu_util_percent": 51.525776397515536, "ram_util_percent": 38.588198757763976}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2770.5642174531786, "episode_reward_min": 863.929739874468, "episode_reward_mean": 1916.13174319548, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.54923230028109, "mean_processing_ms": 1.5307913837053462, "mean_inference_ms": 0.875794920757431}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3773952, "num_steps_sampled": 3780000, "sample_time_ms": 235193.279, "load_time_ms": 10.913, "grad_time_ms": 5386.77, "update_time_ms": 4.484, "learner": {"default_policy": {"cur_kl_coeff": 2.842170985392048e-15, "cur_lr": 4.999999873689376e-05, "total_loss": 349.8982238769531, "policy_loss": -0.00020913034677505493, "vf_loss": 349.8984375, "vf_explained_var": -4.330252156847791e-09, "kl": 0.002744710072875023, "entropy": -0.6242475509643555, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3780000, "episodes_total": 1260, "training_iteration": 63, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-33-33", "timestamp": 1570779213, "time_this_iter_s": 226.63115096092224, "time_total_s": 15506.138483047485, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 15506.138483047485, "timesteps_since_restore": 3780000, "iterations_since_restore": 63, "perf": {"cpu_util_percent": 51.44303405572755, "ram_util_percent": 38.59380804953561}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2823.208155771354, "episode_reward_min": 950.123074259191, "episode_reward_mean": 2015.9632386375406, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.552790642295822, "mean_processing_ms": 1.5315025825123991, "mean_inference_ms": 0.8767107927471391}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3833856, "num_steps_sampled": 3840000, "sample_time_ms": 235454.431, "load_time_ms": 10.902, "grad_time_ms": 5384.433, "update_time_ms": 4.44, "learner": {"default_policy": {"cur_kl_coeff": 1.421085492696024e-15, "cur_lr": 4.999999873689376e-05, "total_loss": 371.93218994140625, "policy_loss": 9.162112837657332e-05, "vf_loss": 371.9320983886719, "vf_explained_var": -5.858576290762585e-09, "kl": 0.0026667057536542416, "entropy": -0.6451295018196106, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3840000, "episodes_total": 1280, "training_iteration": 64, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-37-21", "timestamp": 1570779441, "time_this_iter_s": 227.52810668945312, "time_total_s": 15733.666589736938, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 15733.666589736938, "timesteps_since_restore": 3840000, "iterations_since_restore": 64, "perf": {"cpu_util_percent": 51.15432098765432, "ram_util_percent": 38.59135802469137}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2838.3437090532175, "episode_reward_min": 1180.449795868128, "episode_reward_mean": 2140.6203704910176, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.554262030390675, "mean_processing_ms": 1.5314606005149167, "mean_inference_ms": 0.8771664760476363}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3893760, "num_steps_sampled": 3900000, "sample_time_ms": 235890.087, "load_time_ms": 10.893, "grad_time_ms": 5407.627, "update_time_ms": 4.526, "learner": {"default_policy": {"cur_kl_coeff": 7.10542746348012e-16, "cur_lr": 4.999999873689376e-05, "total_loss": 394.1192932128906, "policy_loss": 0.00043883052421733737, "vf_loss": 394.1188659667969, "vf_explained_var": -2.801927800888393e-09, "kl": 0.0025074041914194822, "entropy": -0.6558175086975098, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3900000, "episodes_total": 1300, "training_iteration": 65, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-41-09", "timestamp": 1570779669, "time_this_iter_s": 227.74777269363403, "time_total_s": 15961.414362430573, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 15961.414362430573, "timesteps_since_restore": 3900000, "iterations_since_restore": 65, "perf": {"cpu_util_percent": 51.29723076923078, "ram_util_percent": 38.711384615384624}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2918.3469151650456, "episode_reward_min": 1180.449795868128, "episode_reward_mean": 2161.678808502187, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.547604467399634, "mean_processing_ms": 1.5300211382251143, "mean_inference_ms": 0.8759583229185713}, "off_policy_estimator": {}, "info": {"num_steps_trained": 3953664, "num_steps_sampled": 3960000, "sample_time_ms": 235956.784, "load_time_ms": 10.882, "grad_time_ms": 5405.598, "update_time_ms": 4.527, "learner": {"default_policy": {"cur_kl_coeff": 3.55271373174006e-16, "cur_lr": 4.999999873689376e-05, "total_loss": 356.43096923828125, "policy_loss": -9.269426664104685e-05, "vf_loss": 356.43109130859375, "vf_explained_var": -5.858576290762585e-09, "kl": 0.004095445387065411, "entropy": -0.6566576957702637, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 3960000, "episodes_total": 1320, "training_iteration": 66, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-44-58", "timestamp": 1570779898, "time_this_iter_s": 229.58401441574097, "time_total_s": 16190.998376846313, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 16190.998376846313, "timesteps_since_restore": 3960000, "iterations_since_restore": 66, "perf": {"cpu_util_percent": 51.70823170731707, "ram_util_percent": 40.29725609756098}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2918.3469151650456, "episode_reward_min": 1449.8269873641468, "episode_reward_mean": 2190.5720489630153, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.5413503492006795, "mean_processing_ms": 1.52867746695693, "mean_inference_ms": 0.8748179088503596}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4013568, "num_steps_sampled": 4020000, "sample_time_ms": 235936.016, "load_time_ms": 10.937, "grad_time_ms": 5404.571, "update_time_ms": 4.591, "learner": {"default_policy": {"cur_kl_coeff": 1.77635686587003e-16, "cur_lr": 4.999999873689376e-05, "total_loss": 361.1904296875, "policy_loss": 2.4176051738322712e-05, "vf_loss": 361.1904296875, "vf_explained_var": -5.858576290762585e-09, "kl": 0.002447907580062747, "entropy": -0.6736043095588684, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4020000, "episodes_total": 1340, "training_iteration": 67, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-48-47", "timestamp": 1570780127, "time_this_iter_s": 228.4135136604309, "time_total_s": 16419.411890506744, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 16419.411890506744, "timesteps_since_restore": 4020000, "iterations_since_restore": 67, "perf": {"cpu_util_percent": 51.61134969325153, "ram_util_percent": 40.223312883435575}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2918.3469151650456, "episode_reward_min": 1438.7104641756998, "episode_reward_mean": 2199.050464742624, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.53538874765602, "mean_processing_ms": 1.5273949679781345, "mean_inference_ms": 0.8737303694701037}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4073472, "num_steps_sampled": 4080000, "sample_time_ms": 236330.48, "load_time_ms": 10.922, "grad_time_ms": 5401.914, "update_time_ms": 4.549, "learner": {"default_policy": {"cur_kl_coeff": 8.88178432935015e-17, "cur_lr": 4.999999873689376e-05, "total_loss": 359.51202392578125, "policy_loss": 0.0005820075166411698, "vf_loss": 359.51141357421875, "vf_explained_var": -3.5660898678457897e-09, "kl": 0.0026870097499340773, "entropy": -0.681388795375824, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4080000, "episodes_total": 1360, "training_iteration": 68, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-52-35", "timestamp": 1570780355, "time_this_iter_s": 227.9651095867157, "time_total_s": 16647.37700009346, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 16647.37700009346, "timesteps_since_restore": 4080000, "iterations_since_restore": 68, "perf": {"cpu_util_percent": 51.61538461538461, "ram_util_percent": 40.19753846153845}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2918.3469151650456, "episode_reward_min": 1438.7104641756998, "episode_reward_mean": 2180.1063424663284, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.52963380857221, "mean_processing_ms": 1.5261190507748024, "mean_inference_ms": 0.8726946942863725}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4133376, "num_steps_sampled": 4140000, "sample_time_ms": 236016.01, "load_time_ms": 10.895, "grad_time_ms": 5391.981, "update_time_ms": 4.601, "learner": {"default_policy": {"cur_kl_coeff": 4.440892164675075e-17, "cur_lr": 4.999999873689376e-05, "total_loss": 334.80535888671875, "policy_loss": -0.00015563257329631597, "vf_loss": 334.80548095703125, "vf_explained_var": -2.801927800888393e-09, "kl": 0.0039174663834273815, "entropy": -0.6866528391838074, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4140000, "episodes_total": 1380, "training_iteration": 69, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_00-56-22", "timestamp": 1570780582, "time_this_iter_s": 227.43166851997375, "time_total_s": 16874.808668613434, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 16874.808668613434, "timesteps_since_restore": 4140000, "iterations_since_restore": 69, "perf": {"cpu_util_percent": 51.285802469135795, "ram_util_percent": 40.20092592592592}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2918.3469151650456, "episode_reward_min": 1234.8646683984796, "episode_reward_mean": 2141.4014661967985, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.524144605373924, "mean_processing_ms": 1.5249343304998597, "mean_inference_ms": 0.8717333269251171}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4193280, "num_steps_sampled": 4200000, "sample_time_ms": 233176.765, "load_time_ms": 10.321, "grad_time_ms": 5039.15, "update_time_ms": 4.554, "learner": {"default_policy": {"cur_kl_coeff": 2.2204460823375376e-17, "cur_lr": 4.999999873689376e-05, "total_loss": 321.6651611328125, "policy_loss": -0.0004485242534428835, "vf_loss": 321.66558837890625, "vf_explained_var": 2.5472071674137453e-10, "kl": 0.0034829562064260244, "entropy": -0.6966060996055603, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4200000, "episodes_total": 1400, "training_iteration": 70, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-00-10", "timestamp": 1570780810, "time_this_iter_s": 228.37273383140564, "time_total_s": 17103.18140244484, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 17103.18140244484, "timesteps_since_restore": 4200000, "iterations_since_restore": 70, "perf": {"cpu_util_percent": 51.54509202453987, "ram_util_percent": 40.19938650306748}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2980.2579841057523, "episode_reward_min": 1234.8646683984796, "episode_reward_mean": 2194.1291903129827, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.518825705227784, "mean_processing_ms": 1.5236431873458554, "mean_inference_ms": 0.8707753273153799}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4253184, "num_steps_sampled": 4260000, "sample_time_ms": 222703.795, "load_time_ms": 10.323, "grad_time_ms": 5042.358, "update_time_ms": 4.515, "learner": {"default_policy": {"cur_kl_coeff": 1.1102230411687688e-17, "cur_lr": 4.999999873689376e-05, "total_loss": 442.0274963378906, "policy_loss": 0.00017301113985013217, "vf_loss": 442.02734375, "vf_explained_var": -3.1840088343670914e-09, "kl": 0.0018270820146426558, "entropy": -0.7168861031532288, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4260000, "episodes_total": 1420, "training_iteration": 71, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-03-59", "timestamp": 1570781039, "time_this_iter_s": 228.53736877441406, "time_total_s": 17331.718771219254, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 17331.718771219254, "timesteps_since_restore": 4260000, "iterations_since_restore": 71, "perf": {"cpu_util_percent": 51.372699386503065, "ram_util_percent": 40.19999999999999}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2980.2579841057523, "episode_reward_min": 1234.8646683984796, "episode_reward_mean": 2218.0602102754488, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.51367882214365, "mean_processing_ms": 1.5223507510953542, "mean_inference_ms": 0.8698296614092382}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4313088, "num_steps_sampled": 4320000, "sample_time_ms": 223032.386, "load_time_ms": 10.339, "grad_time_ms": 5051.127, "update_time_ms": 4.604, "learner": {"default_policy": {"cur_kl_coeff": 5.551115205843844e-18, "cur_lr": 4.999999873689376e-05, "total_loss": 401.68597412109375, "policy_loss": -0.0001589663588674739, "vf_loss": 401.6861572265625, "vf_explained_var": -2.9292881453812925e-09, "kl": 0.003532812464982271, "entropy": -0.7197285294532776, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4320000, "episodes_total": 1440, "training_iteration": 72, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-07-48", "timestamp": 1570781268, "time_this_iter_s": 228.91155910491943, "time_total_s": 17560.630330324173, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 17560.630330324173, "timesteps_since_restore": 4320000, "iterations_since_restore": 72, "perf": {"cpu_util_percent": 51.31009174311927, "ram_util_percent": 40.27033639143731}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2980.2579841057523, "episode_reward_min": 1234.8646683984796, "episode_reward_mean": 2257.353306603134, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.508814184716648, "mean_processing_ms": 1.5210246095044329, "mean_inference_ms": 0.8689136654499581}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4372992, "num_steps_sampled": 4380000, "sample_time_ms": 223295.858, "load_time_ms": 10.336, "grad_time_ms": 5055.486, "update_time_ms": 4.529, "learner": {"default_policy": {"cur_kl_coeff": 2.775557602921922e-18, "cur_lr": 4.999999873689376e-05, "total_loss": 424.34722900390625, "policy_loss": 0.0001054785680025816, "vf_loss": 424.3470764160156, "vf_explained_var": -3.82081061234274e-10, "kl": 0.0029879065696150064, "entropy": -0.7365134358406067, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4380000, "episodes_total": 1460, "training_iteration": 73, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-11-37", "timestamp": 1570781497, "time_this_iter_s": 229.3085606098175, "time_total_s": 17789.93889093399, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 17789.93889093399, "timesteps_since_restore": 4380000, "iterations_since_restore": 73, "perf": {"cpu_util_percent": 51.407339449541276, "ram_util_percent": 40.28990825688074}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2980.2579841057523, "episode_reward_min": 1234.8646683984796, "episode_reward_mean": 2268.0253831030195, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.504433923819293, "mean_processing_ms": 1.5197335292932104, "mean_inference_ms": 0.8680703813076894}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4432896, "num_steps_sampled": 4440000, "sample_time_ms": 223670.458, "load_time_ms": 10.328, "grad_time_ms": 5059.616, "update_time_ms": 4.528, "learner": {"default_policy": {"cur_kl_coeff": 1.387778801460961e-18, "cur_lr": 4.999999873689376e-05, "total_loss": 352.5968933105469, "policy_loss": 2.7753349058912136e-05, "vf_loss": 352.59686279296875, "vf_explained_var": -7.896342246738186e-09, "kl": 0.003407558426260948, "entropy": -0.7281766533851624, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4440000, "episodes_total": 1480, "training_iteration": 74, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-15-28", "timestamp": 1570781728, "time_this_iter_s": 231.31575798988342, "time_total_s": 18021.254648923874, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 18021.254648923874, "timesteps_since_restore": 4440000, "iterations_since_restore": 74, "perf": {"cpu_util_percent": 53.858787878787886, "ram_util_percent": 40.31060606060606}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2980.2579841057523, "episode_reward_min": 1324.2961702543648, "episode_reward_mean": 2302.4423185331275, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.5002274266484985, "mean_processing_ms": 1.5184109592396267, "mean_inference_ms": 0.8672434217662768}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4492800, "num_steps_sampled": 4500000, "sample_time_ms": 223780.011, "load_time_ms": 10.328, "grad_time_ms": 5049.016, "update_time_ms": 4.453, "learner": {"default_policy": {"cur_kl_coeff": 6.938894007304805e-19, "cur_lr": 4.999999873689376e-05, "total_loss": 384.5096130371094, "policy_loss": 0.00026803341461345553, "vf_loss": 384.50933837890625, "vf_explained_var": -1.7830449339228949e-09, "kl": 0.002893443452194333, "entropy": -0.7438187599182129, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4500000, "episodes_total": 1500, "training_iteration": 75, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-19-17", "timestamp": 1570781957, "time_this_iter_s": 228.7355146408081, "time_total_s": 18249.990163564682, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 18249.990163564682, "timesteps_since_restore": 4500000, "iterations_since_restore": 75, "perf": {"cpu_util_percent": 51.395092024539885, "ram_util_percent": 40.45368098159509}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2911.83179233421, "episode_reward_min": 1324.2961702543648, "episode_reward_mean": 2266.767449209455, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.495962339521445, "mean_processing_ms": 1.5170809035207744, "mean_inference_ms": 0.8664305911897956}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4552704, "num_steps_sampled": 4560000, "sample_time_ms": 223367.646, "load_time_ms": 10.326, "grad_time_ms": 5033.152, "update_time_ms": 4.436, "learner": {"default_policy": {"cur_kl_coeff": 3.4694470036524025e-19, "cur_lr": 4.999999873689376e-05, "total_loss": 372.38580322265625, "policy_loss": 0.0002955401432700455, "vf_loss": 372.385498046875, "vf_explained_var": -2.801927800888393e-09, "kl": 0.002627121051773429, "entropy": -0.7563161849975586, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4560000, "episodes_total": 1520, "training_iteration": 76, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-23-03", "timestamp": 1570782183, "time_this_iter_s": 225.30077481269836, "time_total_s": 18475.29093837738, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 18475.29093837738, "timesteps_since_restore": 4560000, "iterations_since_restore": 76, "perf": {"cpu_util_percent": 51.52298136645963, "ram_util_percent": 40.475776397515524}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2884.4544596348105, "episode_reward_min": 1324.2961702543648, "episode_reward_mean": 2236.2977243007317, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.491742300035577, "mean_processing_ms": 1.5157512748819035, "mean_inference_ms": 0.8656393972631606}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4612608, "num_steps_sampled": 4620000, "sample_time_ms": 223242.146, "load_time_ms": 10.313, "grad_time_ms": 5026.954, "update_time_ms": 4.398, "learner": {"default_policy": {"cur_kl_coeff": 1.7347235018262012e-19, "cur_lr": 4.999999873689376e-05, "total_loss": 344.80633544921875, "policy_loss": -0.0005085475277155638, "vf_loss": 344.80682373046875, "vf_explained_var": -2.0377657339309962e-09, "kl": 0.0059786452911794186, "entropy": -0.7594984173774719, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4620000, "episodes_total": 1540, "training_iteration": 77, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-26-50", "timestamp": 1570782410, "time_this_iter_s": 227.0959987640381, "time_total_s": 18702.38693714142, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 18702.38693714142, "timesteps_since_restore": 4620000, "iterations_since_restore": 77, "perf": {"cpu_util_percent": 51.49907407407407, "ram_util_percent": 40.492901234567896}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3006.42743083957, "episode_reward_min": 1324.2961702543648, "episode_reward_mean": 2177.081601117421, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.487453627166375, "mean_processing_ms": 1.5145325896901238, "mean_inference_ms": 0.8648715494975473}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4672512, "num_steps_sampled": 4680000, "sample_time_ms": 223225.799, "load_time_ms": 10.244, "grad_time_ms": 5030.901, "update_time_ms": 4.373, "learner": {"default_policy": {"cur_kl_coeff": 8.673617509131006e-20, "cur_lr": 4.999999873689376e-05, "total_loss": 319.20611572265625, "policy_loss": 0.0007855209405533969, "vf_loss": 319.205322265625, "vf_explained_var": 2.292486422916795e-09, "kl": 0.002575932303443551, "entropy": -0.7642051577568054, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4680000, "episodes_total": 1560, "training_iteration": 78, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-30-38", "timestamp": 1570782638, "time_this_iter_s": 227.83968496322632, "time_total_s": 18930.226622104645, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 18930.226622104645, "timesteps_since_restore": 4680000, "iterations_since_restore": 78, "perf": {"cpu_util_percent": 51.61015384615384, "ram_util_percent": 40.49446153846154}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3006.42743083957, "episode_reward_min": 1324.2961702543648, "episode_reward_mean": 2189.8382442649468, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.482979743863555, "mean_processing_ms": 1.5133961910688865, "mean_inference_ms": 0.8640697710362111}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4732416, "num_steps_sampled": 4740000, "sample_time_ms": 223224.944, "load_time_ms": 10.25, "grad_time_ms": 5037.512, "update_time_ms": 4.403, "learner": {"default_policy": {"cur_kl_coeff": 4.336808754565503e-20, "cur_lr": 4.999999873689376e-05, "total_loss": 372.6768798828125, "policy_loss": -0.00024281104560941458, "vf_loss": 372.6771545410156, "vf_explained_var": -7.641621557752387e-09, "kl": 0.003701510140672326, "entropy": -0.7932894229888916, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4740000, "episodes_total": 1580, "training_iteration": 79, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-34-25", "timestamp": 1570782865, "time_this_iter_s": 227.48921823501587, "time_total_s": 19157.71584033966, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 19157.71584033966, "timesteps_since_restore": 4740000, "iterations_since_restore": 79, "perf": {"cpu_util_percent": 51.58364197530864, "ram_util_percent": 40.49413580246914}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3006.42743083957, "episode_reward_min": 1452.764198488252, "episode_reward_mean": 2157.248776174541, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.4784927745283305, "mean_processing_ms": 1.5122587457884216, "mean_inference_ms": 0.863272640111636}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4792320, "num_steps_sampled": 4800000, "sample_time_ms": 223018.747, "load_time_ms": 10.185, "grad_time_ms": 5038.609, "update_time_ms": 4.295, "learner": {"default_policy": {"cur_kl_coeff": 2.1684043772827515e-20, "cur_lr": 4.999999873689376e-05, "total_loss": 326.7668151855469, "policy_loss": -1.17258769023465e-05, "vf_loss": 326.766845703125, "vf_explained_var": -1.9104053894380968e-09, "kl": 0.004748365841805935, "entropy": -0.7888689041137695, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4800000, "episodes_total": 1600, "training_iteration": 80, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-38-11", "timestamp": 1570783091, "time_this_iter_s": 226.3192265033722, "time_total_s": 19384.035066843033, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 19384.035066843033, "timesteps_since_restore": 4800000, "iterations_since_restore": 80, "perf": {"cpu_util_percent": 51.45665634674923, "ram_util_percent": 40.49752321981424}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3006.42743083957, "episode_reward_min": 1332.1297631757272, "episode_reward_mean": 2183.5909989627335, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.474336296065456, "mean_processing_ms": 1.5113205798127114, "mean_inference_ms": 0.8625214706630987}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4852224, "num_steps_sampled": 4860000, "sample_time_ms": 223159.356, "load_time_ms": 10.118, "grad_time_ms": 5048.353, "update_time_ms": 4.8, "learner": {"default_policy": {"cur_kl_coeff": 1.0842021886413758e-20, "cur_lr": 4.999999873689376e-05, "total_loss": 413.1332702636719, "policy_loss": -0.0002729884581640363, "vf_loss": 413.1335754394531, "vf_explained_var": -6.622738357719982e-09, "kl": 0.0043872338719666, "entropy": -0.8144399523735046, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4860000, "episodes_total": 1620, "training_iteration": 81, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-42-01", "timestamp": 1570783321, "time_this_iter_s": 230.04490852355957, "time_total_s": 19614.079975366592, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 19614.079975366592, "timesteps_since_restore": 4860000, "iterations_since_restore": 81, "perf": {"cpu_util_percent": 51.63384146341464, "ram_util_percent": 40.49603658536586}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3006.42743083957, "episode_reward_min": 1332.1297631757272, "episode_reward_mean": 2234.8340329616385, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.470398736536834, "mean_processing_ms": 1.5104265040217797, "mean_inference_ms": 0.8617891469656084}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4912128, "num_steps_sampled": 4920000, "sample_time_ms": 223228.435, "load_time_ms": 10.089, "grad_time_ms": 5043.321, "update_time_ms": 4.774, "learner": {"default_policy": {"cur_kl_coeff": 5.421010943206879e-21, "cur_lr": 4.999999873689376e-05, "total_loss": 426.8544921875, "policy_loss": -0.00012361582776065916, "vf_loss": 426.8546447753906, "vf_explained_var": -5.094414223805188e-09, "kl": 0.0035946981515735388, "entropy": -0.824469268321991, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4920000, "episodes_total": 1640, "training_iteration": 82, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-45-51", "timestamp": 1570783551, "time_this_iter_s": 229.55120015144348, "time_total_s": 19843.631175518036, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 19843.631175518036, "timesteps_since_restore": 4920000, "iterations_since_restore": 82, "perf": {"cpu_util_percent": 51.297256097560975, "ram_util_percent": 40.5}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2933.967885678605, "episode_reward_min": 1332.1297631757272, "episode_reward_mean": 2219.425534777116, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.4664105812545, "mean_processing_ms": 1.5094611079483513, "mean_inference_ms": 0.8610582265066498}, "off_policy_estimator": {}, "info": {"num_steps_trained": 4972032, "num_steps_sampled": 4980000, "sample_time_ms": 222835.053, "load_time_ms": 10.041, "grad_time_ms": 5045.128, "update_time_ms": 4.84, "learner": {"default_policy": {"cur_kl_coeff": 2.7105054716034394e-21, "cur_lr": 4.999999873689376e-05, "total_loss": 298.0599060058594, "policy_loss": -0.00013286175089888275, "vf_loss": 298.0600280761719, "vf_explained_var": -4.202891812354892e-09, "kl": 0.0036338898353278637, "entropy": -0.7899288535118103, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 4980000, "episodes_total": 1660, "training_iteration": 83, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-49-36", "timestamp": 1570783776, "time_this_iter_s": 225.3933732509613, "time_total_s": 20069.024548768997, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 20069.024548768997, "timesteps_since_restore": 4980000, "iterations_since_restore": 83, "perf": {"cpu_util_percent": 51.38411214953271, "ram_util_percent": 40.498753894081}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2916.691916413175, "episode_reward_min": 1332.1297631757272, "episode_reward_mean": 2202.864556787631, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.462527868617465, "mean_processing_ms": 1.508453193103734, "mean_inference_ms": 0.8603528209390059}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5031936, "num_steps_sampled": 5040000, "sample_time_ms": 222449.213, "load_time_ms": 10.05, "grad_time_ms": 5049.473, "update_time_ms": 4.829, "learner": {"default_policy": {"cur_kl_coeff": 1.3552527358017197e-21, "cur_lr": 4.999999873689376e-05, "total_loss": 334.1376953125, "policy_loss": -0.00046916879364289343, "vf_loss": 334.1381530761719, "vf_explained_var": -4.457612501340691e-09, "kl": 0.00451930146664381, "entropy": -0.819091796875, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5040000, "episodes_total": 1680, "training_iteration": 84, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-53-24", "timestamp": 1570784004, "time_this_iter_s": 227.5010061264038, "time_total_s": 20296.5255548954, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 20296.5255548954, "timesteps_since_restore": 5040000, "iterations_since_restore": 84, "perf": {"cpu_util_percent": 51.315384615384616, "ram_util_percent": 40.49846153846154}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2905.747602907159, "episode_reward_min": 1332.1297631757272, "episode_reward_mean": 2179.710890083026, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.4587606544091845, "mean_processing_ms": 1.5075156912487153, "mean_inference_ms": 0.8596746607010859}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5091840, "num_steps_sampled": 5100000, "sample_time_ms": 222331.737, "load_time_ms": 10.059, "grad_time_ms": 5039.01, "update_time_ms": 4.823, "learner": {"default_policy": {"cur_kl_coeff": 6.776263679008599e-22, "cur_lr": 4.999999873689376e-05, "total_loss": 283.073974609375, "policy_loss": 0.0002259069588035345, "vf_loss": 283.07373046875, "vf_explained_var": -1.1080350859060673e-08, "kl": 0.0033177239820361137, "entropy": -0.8060696721076965, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5100000, "episodes_total": 1700, "training_iteration": 85, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_01-57-11", "timestamp": 1570784231, "time_this_iter_s": 227.45634031295776, "time_total_s": 20523.98189520836, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 20523.98189520836, "timesteps_since_restore": 5100000, "iterations_since_restore": 85, "perf": {"cpu_util_percent": 51.41851851851851, "ram_util_percent": 40.498765432098764}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2905.747602907159, "episode_reward_min": 1196.2430041476587, "episode_reward_mean": 2104.488545126206, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.454831173362524, "mean_processing_ms": 1.5064578514085258, "mean_inference_ms": 0.8589962138263258}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5151744, "num_steps_sampled": 5160000, "sample_time_ms": 222324.584, "load_time_ms": 10.02, "grad_time_ms": 5038.046, "update_time_ms": 4.813, "learner": {"default_policy": {"cur_kl_coeff": 3.3881318395042993e-22, "cur_lr": 4.999999873689376e-05, "total_loss": 291.6480407714844, "policy_loss": -4.706958497990854e-05, "vf_loss": 291.6481018066406, "vf_explained_var": -3.056648489874192e-09, "kl": 0.003895676461979747, "entropy": -0.8070542812347412, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5160000, "episodes_total": 1720, "training_iteration": 86, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-00-57", "timestamp": 1570784457, "time_this_iter_s": 225.21863865852356, "time_total_s": 20749.200533866882, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 20749.200533866882, "timesteps_since_restore": 5160000, "iterations_since_restore": 86, "perf": {"cpu_util_percent": 51.46323987538941, "ram_util_percent": 40.49906542056075}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2899.148447687858, "episode_reward_min": 1196.2430041476587, "episode_reward_mean": 2052.3106909380394, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.450893973983236, "mean_processing_ms": 1.5054460582614069, "mean_inference_ms": 0.8583462908434553}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5211648, "num_steps_sampled": 5220000, "sample_time_ms": 222378.112, "load_time_ms": 9.955, "grad_time_ms": 5040.267, "update_time_ms": 4.804, "learner": {"default_policy": {"cur_kl_coeff": 1.6940659197521496e-22, "cur_lr": 4.999999873689376e-05, "total_loss": 336.4368896484375, "policy_loss": 0.00021401583217084408, "vf_loss": 336.43670654296875, "vf_explained_var": -5.985936635255484e-09, "kl": 0.002751996275037527, "entropy": -0.8360207080841064, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5220000, "episodes_total": 1740, "training_iteration": 87, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-04-44", "timestamp": 1570784684, "time_this_iter_s": 227.65292978286743, "time_total_s": 20976.85346364975, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 20976.85346364975, "timesteps_since_restore": 5220000, "iterations_since_restore": 87, "perf": {"cpu_util_percent": 51.559384615384616, "ram_util_percent": 40.5}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2758.315550076924, "episode_reward_min": 1013.5978424744745, "episode_reward_mean": 2035.4545333821252, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.447115531919371, "mean_processing_ms": 1.5045399413494056, "mean_inference_ms": 0.8577291555253139}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5271552, "num_steps_sampled": 5280000, "sample_time_ms": 222299.236, "load_time_ms": 10.005, "grad_time_ms": 5043.785, "update_time_ms": 4.751, "learner": {"default_policy": {"cur_kl_coeff": 8.470329598760748e-23, "cur_lr": 4.999999873689376e-05, "total_loss": 271.9049072265625, "policy_loss": 0.0003052720567211509, "vf_loss": 271.90460205078125, "vf_explained_var": -1.2226593959496768e-08, "kl": 0.003503115614876151, "entropy": -0.8166009187698364, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5280000, "episodes_total": 1760, "training_iteration": 88, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-08-31", "timestamp": 1570784911, "time_this_iter_s": 227.0863552093506, "time_total_s": 21203.9398188591, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 21203.9398188591, "timesteps_since_restore": 5280000, "iterations_since_restore": 88, "perf": {"cpu_util_percent": 51.49660493827161, "ram_util_percent": 40.5}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2737.0613425937045, "episode_reward_min": 1013.5978424744745, "episode_reward_mean": 2026.995016320426, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.443448861807843, "mean_processing_ms": 1.503688345148238, "mean_inference_ms": 0.8571220444556282}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5331456, "num_steps_sampled": 5340000, "sample_time_ms": 222291.746, "load_time_ms": 9.997, "grad_time_ms": 5040.162, "update_time_ms": 4.672, "learner": {"default_policy": {"cur_kl_coeff": 4.235164799380374e-23, "cur_lr": 4.999999873689376e-05, "total_loss": 331.7037353515625, "policy_loss": -0.000127263818285428, "vf_loss": 331.703857421875, "vf_explained_var": -4.0755314678619925e-09, "kl": 0.004985067993402481, "entropy": -0.8520377278327942, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5340000, "episodes_total": 1780, "training_iteration": 89, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-12-19", "timestamp": 1570785139, "time_this_iter_s": 227.37750267982483, "time_total_s": 21431.317321538925, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 21431.317321538925, "timesteps_since_restore": 5340000, "iterations_since_restore": 89, "perf": {"cpu_util_percent": 51.66461538461539, "ram_util_percent": 40.5}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2737.0613425937045, "episode_reward_min": 964.4086085056584, "episode_reward_mean": 1999.0844661466292, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.443818108807803, "mean_processing_ms": 1.5037491850696432, "mean_inference_ms": 0.8573441394371691}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5391360, "num_steps_sampled": 5400000, "sample_time_ms": 230063.943, "load_time_ms": 10.431, "grad_time_ms": 5466.461, "update_time_ms": 4.782, "learner": {"default_policy": {"cur_kl_coeff": 2.117582399690187e-23, "cur_lr": 4.999999873689376e-05, "total_loss": 252.10047912597656, "policy_loss": -0.00036170537350699306, "vf_loss": 252.10084533691406, "vf_explained_var": -2.6745674563954935e-09, "kl": 0.0037506765220314264, "entropy": -0.8334175944328308, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5400000, "episodes_total": 1800, "training_iteration": 90, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-17-27", "timestamp": 1570785447, "time_this_iter_s": 308.3288314342499, "time_total_s": 21739.646152973175, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 21739.646152973175, "timesteps_since_restore": 5400000, "iterations_since_restore": 90, "perf": {"cpu_util_percent": 67.48268792710707, "ram_util_percent": 48.08815489749431}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2737.0613425937045, "episode_reward_min": 964.4086085056584, "episode_reward_mean": 1962.2322116681294, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.449222481375564, "mean_processing_ms": 1.504937074639416, "mean_inference_ms": 0.8585842609888559}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5451264, "num_steps_sampled": 5460000, "sample_time_ms": 239495.747, "load_time_ms": 10.873, "grad_time_ms": 5756.357, "update_time_ms": 4.534, "learner": {"default_policy": {"cur_kl_coeff": 1.0587911998450935e-23, "cur_lr": 4.999999873689376e-05, "total_loss": 230.7174072265625, "policy_loss": 0.0004092047456651926, "vf_loss": 230.71697998046875, "vf_explained_var": -7.641621557752387e-09, "kl": 0.0024183830246329308, "entropy": -0.8498794436454773, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5460000, "episodes_total": 1820, "training_iteration": 91, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-22-54", "timestamp": 1570785774, "time_this_iter_s": 327.270468711853, "time_total_s": 22066.916621685028, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 22066.916621685028, "timesteps_since_restore": 5460000, "iterations_since_restore": 91, "perf": {"cpu_util_percent": 69.03276231263382, "ram_util_percent": 52.65867237687366}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2677.557837368173, "episode_reward_min": 964.4086085056584, "episode_reward_mean": 1940.4812706349296, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.458546806651846, "mean_processing_ms": 1.507038553790822, "mean_inference_ms": 0.8606399314970993}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5511168, "num_steps_sampled": 5520000, "sample_time_ms": 247162.509, "load_time_ms": 10.884, "grad_time_ms": 5753.814, "update_time_ms": 4.579, "learner": {"default_policy": {"cur_kl_coeff": 5.293955999225468e-24, "cur_lr": 4.999999873689376e-05, "total_loss": 297.43548583984375, "policy_loss": -0.00034542870707809925, "vf_loss": 297.435791015625, "vf_explained_var": -4.71233319032649e-09, "kl": 0.004611298441886902, "entropy": -0.8941606283187866, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5520000, "episodes_total": 1840, "training_iteration": 92, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-28-01", "timestamp": 1570786081, "time_this_iter_s": 306.19391202926636, "time_total_s": 22373.110533714294, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 22373.110533714294, "timesteps_since_restore": 5520000, "iterations_since_restore": 92, "perf": {"cpu_util_percent": 65.68878718535468, "ram_util_percent": 52.97528604118993}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2683.5308525802825, "episode_reward_min": 964.4086085056584, "episode_reward_mean": 1957.650974117936, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.467778906102963, "mean_processing_ms": 1.5091149986232149, "mean_inference_ms": 0.8626515789327578}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5571072, "num_steps_sampled": 5580000, "sample_time_ms": 247183.82, "load_time_ms": 10.993, "grad_time_ms": 5749.685, "update_time_ms": 4.529, "learner": {"default_policy": {"cur_kl_coeff": 2.646977999612734e-24, "cur_lr": 4.999999873689376e-05, "total_loss": 290.9716796875, "policy_loss": -0.0004666796594392508, "vf_loss": 290.97216796875, "vf_explained_var": -6.750098702212881e-09, "kl": 0.0057234009727835655, "entropy": -0.8967065811157227, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5580000, "episodes_total": 1860, "training_iteration": 93, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-31-46", "timestamp": 1570786306, "time_this_iter_s": 225.56541466712952, "time_total_s": 22598.675948381424, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 22598.675948381424, "timesteps_since_restore": 5580000, "iterations_since_restore": 93, "perf": {"cpu_util_percent": 51.570093457943926, "ram_util_percent": 46.34517133956386}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2683.5308525802825, "episode_reward_min": 964.4086085056584, "episode_reward_mean": 1939.0772353866698, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.47675565317096, "mean_processing_ms": 1.5111543505451976, "mean_inference_ms": 0.8646343724194111}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5630976, "num_steps_sampled": 5640000, "sample_time_ms": 246921.379, "load_time_ms": 11.012, "grad_time_ms": 5738.348, "update_time_ms": 4.572, "learner": {"default_policy": {"cur_kl_coeff": 1.323488999806367e-24, "cur_lr": 4.999999873689376e-05, "total_loss": 290.494873046875, "policy_loss": 0.00038343845517374575, "vf_loss": 290.4944763183594, "vf_explained_var": -7.514261213259488e-09, "kl": 0.004202503245323896, "entropy": -0.9158763289451599, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5640000, "episodes_total": 1880, "training_iteration": 94, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-35-31", "timestamp": 1570786531, "time_this_iter_s": 224.76352715492249, "time_total_s": 22823.439475536346, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 22823.439475536346, "timesteps_since_restore": 5640000, "iterations_since_restore": 94, "perf": {"cpu_util_percent": 51.34610591900311, "ram_util_percent": 46.16386292834892}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2861.060956793371, "episode_reward_min": 1256.0643935563871, "episode_reward_mean": 2000.0722688135281, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.481652038325564, "mean_processing_ms": 1.5122527184887817, "mean_inference_ms": 0.8657620794464368}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5690880, "num_steps_sampled": 5700000, "sample_time_ms": 246709.273, "load_time_ms": 10.982, "grad_time_ms": 5737.554, "update_time_ms": 4.54, "learner": {"default_policy": {"cur_kl_coeff": 6.617444999031835e-25, "cur_lr": 4.999999873689376e-05, "total_loss": 329.7452392578125, "policy_loss": 3.619683047872968e-05, "vf_loss": 329.74517822265625, "vf_explained_var": -2.292486422916795e-09, "kl": 0.005399812012910843, "entropy": -0.9454441070556641, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5700000, "episodes_total": 1900, "training_iteration": 95, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-39-16", "timestamp": 1570786756, "time_this_iter_s": 225.32696080207825, "time_total_s": 23048.766436338425, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 23048.766436338425, "timesteps_since_restore": 5700000, "iterations_since_restore": 95, "perf": {"cpu_util_percent": 51.388161993769465, "ram_util_percent": 46.148286604361374}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2861.060956793371, "episode_reward_min": 1290.5253107259348, "episode_reward_mean": 2035.020953785304, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.481518327121863, "mean_processing_ms": 1.5123113546300606, "mean_inference_ms": 0.8658710432810979}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5750784, "num_steps_sampled": 5760000, "sample_time_ms": 246878.635, "load_time_ms": 11.056, "grad_time_ms": 5777.405, "update_time_ms": 4.555, "learner": {"default_policy": {"cur_kl_coeff": 3.3087224995159173e-25, "cur_lr": 4.999999873689376e-05, "total_loss": 286.93316650390625, "policy_loss": -0.0002915517834480852, "vf_loss": 286.9334411621094, "vf_explained_var": -2.9292881453812925e-09, "kl": 0.0048973518423736095, "entropy": -0.9249772429466248, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5760000, "episodes_total": 1920, "training_iteration": 96, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-43-04", "timestamp": 1570786984, "time_this_iter_s": 227.3152379989624, "time_total_s": 23276.081674337387, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 23276.081674337387, "timesteps_since_restore": 5760000, "iterations_since_restore": 96, "perf": {"cpu_util_percent": 51.31784615384616, "ram_util_percent": 46.52492307692307}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2934.826069042796, "episode_reward_min": 1290.5253107259348, "episode_reward_mean": 2063.7627740994685, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.477459937742354, "mean_processing_ms": 1.5114917081981534, "mean_inference_ms": 0.8651604433805758}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5810688, "num_steps_sampled": 5820000, "sample_time_ms": 246975.292, "load_time_ms": 11.151, "grad_time_ms": 5793.693, "update_time_ms": 4.622, "learner": {"default_policy": {"cur_kl_coeff": 1.6543612497579586e-25, "cur_lr": 4.999999873689376e-05, "total_loss": 345.74163818359375, "policy_loss": 0.00039254192961379886, "vf_loss": 345.7412414550781, "vf_explained_var": -7.386900424677378e-09, "kl": 0.00418316712602973, "entropy": -0.9660893678665161, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5820000, "episodes_total": 1940, "training_iteration": 97, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-46-52", "timestamp": 1570787212, "time_this_iter_s": 228.78414392471313, "time_total_s": 23504.8658182621, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 23504.8658182621, "timesteps_since_restore": 5820000, "iterations_since_restore": 97, "perf": {"cpu_util_percent": 51.444478527607366, "ram_util_percent": 47.99110429447853}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2934.826069042796, "episode_reward_min": 1290.5253107259348, "episode_reward_mean": 2093.4478175053637, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.473579763514985, "mean_processing_ms": 1.510672495994402, "mean_inference_ms": 0.8644802404398336}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5870592, "num_steps_sampled": 5880000, "sample_time_ms": 246999.299, "load_time_ms": 11.149, "grad_time_ms": 5787.114, "update_time_ms": 4.698, "learner": {"default_policy": {"cur_kl_coeff": 8.271806248789793e-26, "cur_lr": 4.999999873689376e-05, "total_loss": 329.9728698730469, "policy_loss": -0.0009427159675396979, "vf_loss": 329.9737854003906, "vf_explained_var": -4.839693534819389e-09, "kl": 0.005523227620869875, "entropy": -0.9740684032440186, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5880000, "episodes_total": 1960, "training_iteration": 98, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-50-40", "timestamp": 1570787440, "time_this_iter_s": 227.26147627830505, "time_total_s": 23732.127294540405, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 23732.127294540405, "timesteps_since_restore": 5880000, "iterations_since_restore": 98, "perf": {"cpu_util_percent": 51.58240740740741, "ram_util_percent": 47.92777777777777}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2934.826069042796, "episode_reward_min": 1170.9146099550221, "episode_reward_mean": 2079.6405859727024, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.469870841172653, "mean_processing_ms": 1.5099312980036705, "mean_inference_ms": 0.8638283152478821}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5930496, "num_steps_sampled": 5940000, "sample_time_ms": 246926.178, "load_time_ms": 11.115, "grad_time_ms": 5789.075, "update_time_ms": 4.75, "learner": {"default_policy": {"cur_kl_coeff": 4.1359031243948966e-26, "cur_lr": 4.999999873689376e-05, "total_loss": 276.3343200683594, "policy_loss": 0.00035375356674194336, "vf_loss": 276.3339538574219, "vf_explained_var": -3.1840088343670914e-09, "kl": 0.003470742842182517, "entropy": -0.9474529027938843, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 5940000, "episodes_total": 1980, "training_iteration": 99, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-54-26", "timestamp": 1570787666, "time_this_iter_s": 226.66594099998474, "time_total_s": 23958.79323554039, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 23958.79323554039, "timesteps_since_restore": 5940000, "iterations_since_restore": 99, "perf": {"cpu_util_percent": 51.598765432098766, "ram_util_percent": 47.976851851851855}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2934.826069042796, "episode_reward_min": 1170.9146099550221, "episode_reward_mean": 2059.6971649764732, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.466263041875936, "mean_processing_ms": 1.509240417837348, "mean_inference_ms": 0.8632060163710099}, "off_policy_estimator": {}, "info": {"num_steps_trained": 5990400, "num_steps_sampled": 6000000, "sample_time_ms": 239290.315, "load_time_ms": 10.675, "grad_time_ms": 5361.964, "update_time_ms": 4.705, "learner": {"default_policy": {"cur_kl_coeff": 2.0679515621974483e-26, "cur_lr": 4.999999873689376e-05, "total_loss": 297.5181884765625, "policy_loss": -0.00040703132981434464, "vf_loss": 297.51861572265625, "vf_explained_var": 5.094414334827491e-10, "kl": 0.005609925836324692, "entropy": -0.9748757481575012, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6000000, "episodes_total": 2000, "training_iteration": 100, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_02-58-14", "timestamp": 1570787894, "time_this_iter_s": 227.67542624473572, "time_total_s": 24186.468661785126, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 24186.468661785126, "timesteps_since_restore": 6000000, "iterations_since_restore": 100, "perf": {"cpu_util_percent": 51.16635802469136, "ram_util_percent": 47.998765432098764}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2934.826069042796, "episode_reward_min": 1170.9146099550221, "episode_reward_mean": 2040.987912493069, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.462698785768646, "mean_processing_ms": 1.5085268600760924, "mean_inference_ms": 0.8626049430374765}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6050304, "num_steps_sampled": 6060000, "sample_time_ms": 229393.23, "load_time_ms": 10.251, "grad_time_ms": 5064.066, "update_time_ms": 4.635, "learner": {"default_policy": {"cur_kl_coeff": 1.0339757810987241e-26, "cur_lr": 4.999999873689376e-05, "total_loss": 254.15103149414062, "policy_loss": 0.00040054015698842704, "vf_loss": 254.15060424804688, "vf_explained_var": -1.273603555951297e-09, "kl": 0.0036605566274374723, "entropy": -0.9686458706855774, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6060000, "episodes_total": 2020, "training_iteration": 101, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-01-59", "timestamp": 1570788119, "time_this_iter_s": 225.31001806259155, "time_total_s": 24411.778679847717, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 24411.778679847717, "timesteps_since_restore": 6060000, "iterations_since_restore": 101, "perf": {"cpu_util_percent": 51.42173913043478, "ram_util_percent": 48.000621118012425}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2809.436008132594, "episode_reward_min": 1140.723034175603, "episode_reward_mean": 1995.5661046800012, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.459090960330461, "mean_processing_ms": 1.5077626930834152, "mean_inference_ms": 0.8620029306174886}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6110208, "num_steps_sampled": 6120000, "sample_time_ms": 221298.993, "load_time_ms": 10.239, "grad_time_ms": 5062.694, "update_time_ms": 4.501, "learner": {"default_policy": {"cur_kl_coeff": 5.169878905493621e-27, "cur_lr": 4.999999873689376e-05, "total_loss": 274.35589599609375, "policy_loss": 2.6937734219245613e-05, "vf_loss": 274.3558654785156, "vf_explained_var": -5.985936635255484e-09, "kl": 0.005472501739859581, "entropy": -0.9897372722625732, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6120000, "episodes_total": 2040, "training_iteration": 102, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-05-45", "timestamp": 1570788345, "time_this_iter_s": 225.23614883422852, "time_total_s": 24637.014828681946, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 24637.014828681946, "timesteps_since_restore": 6120000, "iterations_since_restore": 102, "perf": {"cpu_util_percent": 51.4373831775701, "ram_util_percent": 48.06292834890966}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2790.561620110469, "episode_reward_min": 1140.723034175603, "episode_reward_mean": 1941.9584758988715, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.455476650917437, "mean_processing_ms": 1.5070224047497909, "mean_inference_ms": 0.8614178635704232}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6170112, "num_steps_sampled": 6180000, "sample_time_ms": 221370.141, "load_time_ms": 10.144, "grad_time_ms": 5075.015, "update_time_ms": 4.547, "learner": {"default_policy": {"cur_kl_coeff": 2.5849394527468104e-27, "cur_lr": 4.999999873689376e-05, "total_loss": 251.54049682617188, "policy_loss": 9.814197255764157e-06, "vf_loss": 251.54049682617188, "vf_explained_var": -1.0443549136596175e-08, "kl": 0.003990766126662493, "entropy": -0.9907731413841248, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6180000, "episodes_total": 2060, "training_iteration": 103, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-09-31", "timestamp": 1570788571, "time_this_iter_s": 226.39943838119507, "time_total_s": 24863.41426706314, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 24863.41426706314, "timesteps_since_restore": 6180000, "iterations_since_restore": 103, "perf": {"cpu_util_percent": 51.57337461300311, "ram_util_percent": 48.09752321981425}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2730.1414912893997, "episode_reward_min": 1065.1849980808593, "episode_reward_mean": 1917.6636954166001, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.451870524366552, "mean_processing_ms": 1.506245161603624, "mean_inference_ms": 0.860839929966134}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6230016, "num_steps_sampled": 6240000, "sample_time_ms": 221377.192, "load_time_ms": 10.143, "grad_time_ms": 5082.406, "update_time_ms": 4.521, "learner": {"default_policy": {"cur_kl_coeff": 1.2924697263734052e-27, "cur_lr": 4.999999873689376e-05, "total_loss": 244.52688598632812, "policy_loss": 0.00037902427720837295, "vf_loss": 244.52650451660156, "vf_explained_var": -7.386900424677378e-09, "kl": 0.0040008919313549995, "entropy": -0.9825717806816101, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6240000, "episodes_total": 2080, "training_iteration": 104, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-13-16", "timestamp": 1570788796, "time_this_iter_s": 224.90760898590088, "time_total_s": 25088.32187604904, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 25088.32187604904, "timesteps_since_restore": 6240000, "iterations_since_restore": 104, "perf": {"cpu_util_percent": 51.642679127725856, "ram_util_percent": 48.10155763239876}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2656.880583875099, "episode_reward_min": 1065.1849980808593, "episode_reward_mean": 1923.6063345284313, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.4485876723697535, "mean_processing_ms": 1.5055536435757335, "mean_inference_ms": 0.8603106469197712}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6289920, "num_steps_sampled": 6300000, "sample_time_ms": 222031.126, "load_time_ms": 10.243, "grad_time_ms": 5086.015, "update_time_ms": 4.577, "learner": {"default_policy": {"cur_kl_coeff": 6.462348631867026e-28, "cur_lr": 4.999999873689376e-05, "total_loss": 305.9954528808594, "policy_loss": 0.00037878184230066836, "vf_loss": 305.9950866699219, "vf_explained_var": -1.0188828447610376e-08, "kl": 0.0034544565714895725, "entropy": -1.0335384607315063, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6300000, "episodes_total": 2100, "training_iteration": 105, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-17-08", "timestamp": 1570789028, "time_this_iter_s": 231.9039261341095, "time_total_s": 25320.22580218315, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 25320.22580218315, "timesteps_since_restore": 6300000, "iterations_since_restore": 105, "perf": {"cpu_util_percent": 54.30906344410877, "ram_util_percent": 48.616616314199405}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2656.880583875099, "episode_reward_min": 1065.1849980808593, "episode_reward_mean": 1931.7140297846706, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.445487747192605, "mean_processing_ms": 1.5048747807639749, "mean_inference_ms": 0.8598048336385432}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6349824, "num_steps_sampled": 6360000, "sample_time_ms": 222122.505, "load_time_ms": 10.153, "grad_time_ms": 5050.036, "update_time_ms": 4.573, "learner": {"default_policy": {"cur_kl_coeff": 3.231174315933513e-28, "cur_lr": 4.999999873689376e-05, "total_loss": 268.95941162109375, "policy_loss": 0.0002717429306358099, "vf_loss": 268.95916748046875, "vf_explained_var": -9.29730603616008e-09, "kl": 0.0040903775952756405, "entropy": -1.017516851425171, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6360000, "episodes_total": 2120, "training_iteration": 106, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-20-56", "timestamp": 1570789256, "time_this_iter_s": 227.864901304245, "time_total_s": 25548.090703487396, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 25548.090703487396, "timesteps_since_restore": 6360000, "iterations_since_restore": 106, "perf": {"cpu_util_percent": 51.757538461538466, "ram_util_percent": 48.802153846153836}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2656.880583875099, "episode_reward_min": 1065.1849980808593, "episode_reward_mean": 1944.8068969692508, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.442537109313096, "mean_processing_ms": 1.5042352542568365, "mean_inference_ms": 0.8593207328452318}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6409728, "num_steps_sampled": 6420000, "sample_time_ms": 221992.431, "load_time_ms": 10.146, "grad_time_ms": 5037.708, "update_time_ms": 4.367, "learner": {"default_policy": {"cur_kl_coeff": 1.6155871579667565e-28, "cur_lr": 4.999999873689376e-05, "total_loss": 291.67462158203125, "policy_loss": -0.00012354386853985488, "vf_loss": 291.6747131347656, "vf_explained_var": -4.839693534819389e-09, "kl": 0.004860265646129847, "entropy": -1.0397971868515015, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6420000, "episodes_total": 2140, "training_iteration": 107, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-24-43", "timestamp": 1570789483, "time_this_iter_s": 227.35837364196777, "time_total_s": 25775.449077129364, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 25775.449077129364, "timesteps_since_restore": 6420000, "iterations_since_restore": 107, "perf": {"cpu_util_percent": 51.40586419753087, "ram_util_percent": 48.73240740740741}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2712.1523702462287, "episode_reward_min": 1065.1849980808593, "episode_reward_mean": 1943.004044728714, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.439569044549453, "mean_processing_ms": 1.5035642722912863, "mean_inference_ms": 0.8588395121510299}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6469632, "num_steps_sampled": 6480000, "sample_time_ms": 221746.881, "load_time_ms": 10.142, "grad_time_ms": 5036.103, "update_time_ms": 4.383, "learner": {"default_policy": {"cur_kl_coeff": 8.077935789833782e-29, "cur_lr": 4.999999873689376e-05, "total_loss": 249.88047790527344, "policy_loss": 0.0002704112557694316, "vf_loss": 249.8802032470703, "vf_explained_var": -5.349134912790987e-09, "kl": 0.0048729111440479755, "entropy": -1.0350946187973022, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6480000, "episodes_total": 2160, "training_iteration": 108, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-28-28", "timestamp": 1570789708, "time_this_iter_s": 224.78979206085205, "time_total_s": 26000.238869190216, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 26000.238869190216, "timesteps_since_restore": 6480000, "iterations_since_restore": 108, "perf": {"cpu_util_percent": 51.34735202492212, "ram_util_percent": 48.79283489096574}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2712.1523702462287, "episode_reward_min": 1223.9570474460438, "episode_reward_mean": 1953.0073051284094, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.436593404466541, "mean_processing_ms": 1.502886891182002, "mean_inference_ms": 0.8583622739880797}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6529536, "num_steps_sampled": 6540000, "sample_time_ms": 221498.282, "load_time_ms": 10.133, "grad_time_ms": 5035.313, "update_time_ms": 4.325, "learner": {"default_policy": {"cur_kl_coeff": 4.038967894916891e-29, "cur_lr": 4.999999873689376e-05, "total_loss": 245.7689666748047, "policy_loss": 0.00016378675354644656, "vf_loss": 245.76878356933594, "vf_explained_var": -2.1651260784238957e-09, "kl": 0.005801717285066843, "entropy": -1.0496704578399658, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6540000, "episodes_total": 2180, "training_iteration": 109, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-32-12", "timestamp": 1570789932, "time_this_iter_s": 224.17111325263977, "time_total_s": 26224.409982442856, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 26224.409982442856, "timesteps_since_restore": 6540000, "iterations_since_restore": 109, "perf": {"cpu_util_percent": 51.284687500000004, "ram_util_percent": 48.80062500000001}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2772.7075470226778, "episode_reward_min": 1223.9570474460438, "episode_reward_mean": 1954.9965068179956, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.433396673676116, "mean_processing_ms": 1.5021347924239057, "mean_inference_ms": 0.8578479544411897}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6589440, "num_steps_sampled": 6600000, "sample_time_ms": 221391.037, "load_time_ms": 10.122, "grad_time_ms": 5032.943, "update_time_ms": 4.381, "learner": {"default_policy": {"cur_kl_coeff": 2.0194839474584456e-29, "cur_lr": 4.999999873689376e-05, "total_loss": 307.3695983886719, "policy_loss": 0.0003561436606105417, "vf_loss": 307.3692626953125, "vf_explained_var": -2.6745674563954935e-09, "kl": 0.0036990707740187645, "entropy": -1.082297682762146, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6600000, "episodes_total": 2200, "training_iteration": 110, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-35-59", "timestamp": 1570790159, "time_this_iter_s": 226.57998132705688, "time_total_s": 26450.989963769913, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 26450.989963769913, "timesteps_since_restore": 6600000, "iterations_since_restore": 110, "perf": {"cpu_util_percent": 51.28018575851393, "ram_util_percent": 48.86130030959752}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2884.8556156266472, "episode_reward_min": 1223.9570474460438, "episode_reward_mean": 2004.2035484929286, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.430316922296806, "mean_processing_ms": 1.5014319947783252, "mean_inference_ms": 0.8573416353333044}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6649344, "num_steps_sampled": 6660000, "sample_time_ms": 221709.603, "load_time_ms": 10.144, "grad_time_ms": 5030.409, "update_time_ms": 4.287, "learner": {"default_policy": {"cur_kl_coeff": 1.0097419737292228e-29, "cur_lr": 4.999999873689376e-05, "total_loss": 332.7329406738281, "policy_loss": 0.00028941722121089697, "vf_loss": 332.7326965332031, "vf_explained_var": -5.603855601776786e-09, "kl": 0.005987292621284723, "entropy": -1.1064846515655518, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6660000, "episodes_total": 2220, "training_iteration": 111, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-39-47", "timestamp": 1570790387, "time_this_iter_s": 228.4696180820465, "time_total_s": 26679.45958185196, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 26679.45958185196, "timesteps_since_restore": 6660000, "iterations_since_restore": 111, "perf": {"cpu_util_percent": 51.52791411042944, "ram_util_percent": 48.896932515337404}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2940.7799109831026, "episode_reward_min": 1308.0179210304186, "episode_reward_mean": 2055.9638943096447, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.427364009165221, "mean_processing_ms": 1.5007114698388222, "mean_inference_ms": 0.8568438725398819}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6709248, "num_steps_sampled": 6720000, "sample_time_ms": 221980.667, "load_time_ms": 10.181, "grad_time_ms": 5029.406, "update_time_ms": 5.276, "learner": {"default_policy": {"cur_kl_coeff": 5.048709868646114e-30, "cur_lr": 4.999999873689376e-05, "total_loss": 369.8870849609375, "policy_loss": 0.0002594511315692216, "vf_loss": 369.8868103027344, "vf_explained_var": -1.1335071548046471e-08, "kl": 0.004924853798002005, "entropy": -1.1358833312988281, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6720000, "episodes_total": 2240, "training_iteration": 112, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-43-35", "timestamp": 1570790615, "time_this_iter_s": 227.94732356071472, "time_total_s": 26907.406905412674, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 26907.406905412674, "timesteps_since_restore": 6720000, "iterations_since_restore": 112, "perf": {"cpu_util_percent": 51.566769230769225, "ram_util_percent": 48.90523076923076}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2940.7799109831026, "episode_reward_min": 1343.7128477823323, "episode_reward_mean": 2102.4146581586006, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.424558813034257, "mean_processing_ms": 1.500031745123004, "mean_inference_ms": 0.8563517802194234}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6769152, "num_steps_sampled": 6780000, "sample_time_ms": 222069.452, "load_time_ms": 10.224, "grad_time_ms": 5020.797, "update_time_ms": 5.251, "learner": {"default_policy": {"cur_kl_coeff": 2.524354934323057e-30, "cur_lr": 4.999999873689376e-05, "total_loss": 316.38421630859375, "policy_loss": 0.00031902099726721644, "vf_loss": 316.3838806152344, "vf_explained_var": -1.7830449339228949e-09, "kl": 0.0041290284134447575, "entropy": -1.1200151443481445, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6780000, "episodes_total": 2260, "training_iteration": 113, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-47-22", "timestamp": 1570790842, "time_this_iter_s": 227.20156455039978, "time_total_s": 27134.608469963074, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 27134.608469963074, "timesteps_since_restore": 6780000, "iterations_since_restore": 113, "perf": {"cpu_util_percent": 51.27746913580248, "ram_util_percent": 48.99413580246914}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2940.7799109831026, "episode_reward_min": 1121.422021120273, "episode_reward_mean": 2123.797024652828, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.42194423287052, "mean_processing_ms": 1.499433433427161, "mean_inference_ms": 0.8558800953311189}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6829056, "num_steps_sampled": 6840000, "sample_time_ms": 222263.381, "load_time_ms": 10.219, "grad_time_ms": 5020.681, "update_time_ms": 5.247, "learner": {"default_policy": {"cur_kl_coeff": 1.2621774671615285e-30, "cur_lr": 4.999999873689376e-05, "total_loss": 280.3107604980469, "policy_loss": 0.0005952796782366931, "vf_loss": 280.3101501464844, "vf_explained_var": -3.8208107788761936e-09, "kl": 0.004510277882218361, "entropy": -1.1148210763931274, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6840000, "episodes_total": 2280, "training_iteration": 114, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-51-09", "timestamp": 1570791069, "time_this_iter_s": 226.85546731948853, "time_total_s": 27361.463937282562, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 27361.463937282562, "timesteps_since_restore": 6840000, "iterations_since_restore": 114, "perf": {"cpu_util_percent": 51.62376543209877, "ram_util_percent": 48.998765432098764}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2940.7799109831026, "episode_reward_min": 1121.422021120273, "episode_reward_mean": 2142.480614877029, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.4195130422937146, "mean_processing_ms": 1.4988723221142186, "mean_inference_ms": 0.8554391231282957}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6888960, "num_steps_sampled": 6900000, "sample_time_ms": 221949.306, "load_time_ms": 10.093, "grad_time_ms": 5025.647, "update_time_ms": 5.143, "learner": {"default_policy": {"cur_kl_coeff": 6.3108873358076425e-31, "cur_lr": 4.999999873689376e-05, "total_loss": 334.6033020019531, "policy_loss": -8.271870319731534e-05, "vf_loss": 334.6034240722656, "vf_explained_var": 6.368017779756485e-10, "kl": 0.005179248284548521, "entropy": -1.153441309928894, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6900000, "episodes_total": 2300, "training_iteration": 115, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-54-58", "timestamp": 1570791298, "time_this_iter_s": 228.81058764457703, "time_total_s": 27590.27452492714, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 27590.27452492714, "timesteps_since_restore": 6900000, "iterations_since_restore": 115, "perf": {"cpu_util_percent": 51.58006134969326, "ram_util_percent": 49.04141104294479}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2940.7799109831026, "episode_reward_min": 1121.422021120273, "episode_reward_mean": 2119.688029785074, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.41706190677346, "mean_processing_ms": 1.498304746547766, "mean_inference_ms": 0.8550045682509223}, "off_policy_estimator": {}, "info": {"num_steps_trained": 6948864, "num_steps_sampled": 6960000, "sample_time_ms": 221885.029, "load_time_ms": 10.276, "grad_time_ms": 5023.075, "update_time_ms": 5.045, "learner": {"default_policy": {"cur_kl_coeff": 3.1554436679038213e-31, "cur_lr": 4.999999873689376e-05, "total_loss": 299.1117858886719, "policy_loss": 0.00014942610869184136, "vf_loss": 299.111572265625, "vf_explained_var": -7.641621557752387e-09, "kl": 0.0030115745030343533, "entropy": -1.1431351900100708, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 6960000, "episodes_total": 2320, "training_iteration": 116, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_03-58-45", "timestamp": 1570791525, "time_this_iter_s": 227.1993865966797, "time_total_s": 27817.47391152382, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 27817.47391152382, "timesteps_since_restore": 6960000, "iterations_since_restore": 116, "perf": {"cpu_util_percent": 51.53487654320988, "ram_util_percent": 49.10432098765433}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2941.833458681214, "episode_reward_min": 1121.422021120273, "episode_reward_mean": 2090.0029044763705, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.414588395015474, "mean_processing_ms": 1.497791611688474, "mean_inference_ms": 0.8545688636184476}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7008768, "num_steps_sampled": 7020000, "sample_time_ms": 221866.764, "load_time_ms": 10.268, "grad_time_ms": 5019.012, "update_time_ms": 5.107, "learner": {"default_policy": {"cur_kl_coeff": 1.5777218339519106e-31, "cur_lr": 4.999999873689376e-05, "total_loss": 325.9025573730469, "policy_loss": -0.0001517296041129157, "vf_loss": 325.9027099609375, "vf_explained_var": -8.151062935723985e-09, "kl": 0.005776040721684694, "entropy": -1.1616567373275757, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7020000, "episodes_total": 2340, "training_iteration": 117, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-02-32", "timestamp": 1570791752, "time_this_iter_s": 227.13538074493408, "time_total_s": 28044.609292268753, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 28044.609292268753, "timesteps_since_restore": 7020000, "iterations_since_restore": 117, "perf": {"cpu_util_percent": 51.54012345679013, "ram_util_percent": 49.19197530864197}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2941.833458681214, "episode_reward_min": 1121.422021120273, "episode_reward_mean": 2093.3040627928413, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.412179671864106, "mean_processing_ms": 1.497276988574628, "mean_inference_ms": 0.8541445187758996}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7068672, "num_steps_sampled": 7080000, "sample_time_ms": 222092.172, "load_time_ms": 10.369, "grad_time_ms": 5021.272, "update_time_ms": 5.154, "learner": {"default_policy": {"cur_kl_coeff": 7.888609169759553e-32, "cur_lr": 4.999999873689376e-05, "total_loss": 318.9958190917969, "policy_loss": 0.000534237886313349, "vf_loss": 318.99530029296875, "vf_explained_var": -4.202891812354892e-09, "kl": 0.004300462082028389, "entropy": -1.1741962432861328, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7080000, "episodes_total": 2360, "training_iteration": 118, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-06-19", "timestamp": 1570791979, "time_this_iter_s": 227.06834149360657, "time_total_s": 28271.67763376236, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 28271.67763376236, "timesteps_since_restore": 7080000, "iterations_since_restore": 118, "perf": {"cpu_util_percent": 51.39012345679013, "ram_util_percent": 49.09382716049383}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2941.833458681214, "episode_reward_min": 1381.8246345037035, "episode_reward_mean": 2132.742663850571, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.409818033954657, "mean_processing_ms": 1.496748968202069, "mean_inference_ms": 0.8537245284535437}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7128576, "num_steps_sampled": 7140000, "sample_time_ms": 222342.109, "load_time_ms": 10.371, "grad_time_ms": 5019.324, "update_time_ms": 5.17, "learner": {"default_policy": {"cur_kl_coeff": 3.9443045848797766e-32, "cur_lr": 4.999999873689376e-05, "total_loss": 328.4617614746094, "policy_loss": 0.00026661311858333647, "vf_loss": 328.46148681640625, "vf_explained_var": -7.00481939119868e-09, "kl": 0.00501100393012166, "entropy": -1.196395993232727, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7140000, "episodes_total": 2380, "training_iteration": 119, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-10-06", "timestamp": 1570792206, "time_this_iter_s": 226.6509449481964, "time_total_s": 28498.328578710556, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 28498.328578710556, "timesteps_since_restore": 7140000, "iterations_since_restore": 119, "perf": {"cpu_util_percent": 51.58858024691358, "ram_util_percent": 49.12746913580248}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3007.2657019383378, "episode_reward_min": 1381.8246345037035, "episode_reward_mean": 2142.2210019576232, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.407467040365899, "mean_processing_ms": 1.4962414689426806, "mean_inference_ms": 0.8533092066350583}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7188480, "num_steps_sampled": 7200000, "sample_time_ms": 222532.735, "load_time_ms": 10.424, "grad_time_ms": 5027.548, "update_time_ms": 5.139, "learner": {"default_policy": {"cur_kl_coeff": 1.9721522924398883e-32, "cur_lr": 4.999999873689376e-05, "total_loss": 350.4010314941406, "policy_loss": 0.0007466197712346911, "vf_loss": 350.4002990722656, "vf_explained_var": -8.405783624709784e-09, "kl": 0.003580022370442748, "entropy": -1.2104030847549438, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7200000, "episodes_total": 2400, "training_iteration": 120, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-13-55", "timestamp": 1570792435, "time_this_iter_s": 228.56880950927734, "time_total_s": 28726.897388219833, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 28726.897388219833, "timesteps_since_restore": 7200000, "iterations_since_restore": 120, "perf": {"cpu_util_percent": 51.700306748466254, "ram_util_percent": 49.20184049079754}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3035.9625665117865, "episode_reward_min": 1381.8246345037035, "episode_reward_mean": 2188.7578497428026, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.405286999276755, "mean_processing_ms": 1.4957948552218552, "mean_inference_ms": 0.8529035167047238}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7248384, "num_steps_sampled": 7260000, "sample_time_ms": 222843.99, "load_time_ms": 10.438, "grad_time_ms": 5025.774, "update_time_ms": 5.092, "learner": {"default_policy": {"cur_kl_coeff": 9.860761462199441e-33, "cur_lr": 4.999999873689376e-05, "total_loss": 373.27130126953125, "policy_loss": 0.0007099591894075274, "vf_loss": 373.27056884765625, "vf_explained_var": -9.042585347174281e-09, "kl": 0.004816744010895491, "entropy": -1.2301075458526611, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7260000, "episodes_total": 2420, "training_iteration": 121, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-17-46", "timestamp": 1570792666, "time_this_iter_s": 231.5637137889862, "time_total_s": 28958.46110200882, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 28958.46110200882, "timesteps_since_restore": 7260000, "iterations_since_restore": 121, "perf": {"cpu_util_percent": 53.22878787878788, "ram_util_percent": 49.452121212121206}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3057.3607231299898, "episode_reward_min": 1524.9067550215464, "episode_reward_mean": 2208.2048086221653, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.40327161899733, "mean_processing_ms": 1.4953220983408837, "mean_inference_ms": 0.8525253518729536}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7308288, "num_steps_sampled": 7320000, "sample_time_ms": 222980.438, "load_time_ms": 10.443, "grad_time_ms": 5030.24, "update_time_ms": 4.158, "learner": {"default_policy": {"cur_kl_coeff": 4.930380731099721e-33, "cur_lr": 4.999999873689376e-05, "total_loss": 349.9038391113281, "policy_loss": 0.0003130502300336957, "vf_loss": 349.90350341796875, "vf_explained_var": -6.495378013227082e-09, "kl": 0.005526693072170019, "entropy": -1.2306982278823853, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7320000, "episodes_total": 2440, "training_iteration": 122, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-21-36", "timestamp": 1570792896, "time_this_iter_s": 229.34732294082642, "time_total_s": 29187.808424949646, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 29187.808424949646, "timesteps_since_restore": 7320000, "iterations_since_restore": 122, "perf": {"cpu_util_percent": 52.535168195718654, "ram_util_percent": 51.034862385321105}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3057.3607231299898, "episode_reward_min": 1413.2260015039808, "episode_reward_mean": 2212.461681829032, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.401256955778731, "mean_processing_ms": 1.494855449356993, "mean_inference_ms": 0.8521448129227898}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7368192, "num_steps_sampled": 7380000, "sample_time_ms": 222836.759, "load_time_ms": 10.394, "grad_time_ms": 5031.256, "update_time_ms": 4.224, "learner": {"default_policy": {"cur_kl_coeff": 2.4651903655498604e-33, "cur_lr": 4.999999873689376e-05, "total_loss": 324.2953796386719, "policy_loss": -0.0001201698396471329, "vf_loss": 324.29547119140625, "vf_explained_var": -3.056648489874192e-09, "kl": 0.005093560088425875, "entropy": -1.2325332164764404, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7380000, "episodes_total": 2460, "training_iteration": 123, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-25-21", "timestamp": 1570793121, "time_this_iter_s": 225.77511978149414, "time_total_s": 29413.58354473114, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 29413.58354473114, "timesteps_since_restore": 7380000, "iterations_since_restore": 123, "perf": {"cpu_util_percent": 51.517080745341616, "ram_util_percent": 49.963043478260865}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3057.3607231299898, "episode_reward_min": 1321.103810498718, "episode_reward_mean": 2207.9320675315703, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.399314561093524, "mean_processing_ms": 1.4943760066727925, "mean_inference_ms": 0.8517784762022178}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7428096, "num_steps_sampled": 7440000, "sample_time_ms": 222863.773, "load_time_ms": 10.388, "grad_time_ms": 5028.116, "update_time_ms": 4.212, "learner": {"default_policy": {"cur_kl_coeff": 1.2325951827749302e-33, "cur_lr": 4.999999873689376e-05, "total_loss": 327.2377014160156, "policy_loss": 0.0007862969650886953, "vf_loss": 327.2369079589844, "vf_explained_var": -7.896342246738186e-09, "kl": 0.004149282816797495, "entropy": -1.2451683282852173, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7440000, "episodes_total": 2480, "training_iteration": 124, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-29-08", "timestamp": 1570793348, "time_this_iter_s": 227.08454179763794, "time_total_s": 29640.668086528778, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 29640.668086528778, "timesteps_since_restore": 7440000, "iterations_since_restore": 124, "perf": {"cpu_util_percent": 51.53456790123457, "ram_util_percent": 49.998456790123456}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3057.3607231299898, "episode_reward_min": 1321.103810498718, "episode_reward_mean": 2211.0565058699813, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.397341603606959, "mean_processing_ms": 1.493854482234554, "mean_inference_ms": 0.851396021710565}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7488000, "num_steps_sampled": 7500000, "sample_time_ms": 222627.347, "load_time_ms": 10.461, "grad_time_ms": 5026.169, "update_time_ms": 4.391, "learner": {"default_policy": {"cur_kl_coeff": 6.162975913874651e-34, "cur_lr": 4.999999873689376e-05, "total_loss": 349.337158203125, "policy_loss": 0.0010858766036108136, "vf_loss": 349.3360595703125, "vf_explained_var": 1.0188828669654981e-09, "kl": 0.004914838820695877, "entropy": -1.2671468257904053, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7500000, "episodes_total": 2500, "training_iteration": 125, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-32-55", "timestamp": 1570793575, "time_this_iter_s": 226.42987155914307, "time_total_s": 29867.09795808792, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 29867.09795808792, "timesteps_since_restore": 7500000, "iterations_since_restore": 125, "perf": {"cpu_util_percent": 51.38452012383901, "ram_util_percent": 50.013931888544896}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3057.3607231299898, "episode_reward_min": 1321.103810498718, "episode_reward_mean": 2183.928197300355, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.395215937311672, "mean_processing_ms": 1.4932358484074018, "mean_inference_ms": 0.8509987763666014}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7547904, "num_steps_sampled": 7560000, "sample_time_ms": 222465.163, "load_time_ms": 10.33, "grad_time_ms": 5028.049, "update_time_ms": 4.52, "learner": {"default_policy": {"cur_kl_coeff": 3.0814879569373254e-34, "cur_lr": 4.999999873689376e-05, "total_loss": 322.60345458984375, "policy_loss": 8.859772060532123e-05, "vf_loss": 322.6033630371094, "vf_explained_var": -1.273603555951297e-09, "kl": 0.005859797820448875, "entropy": -1.270148515701294, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7560000, "episodes_total": 2520, "training_iteration": 126, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-36-40", "timestamp": 1570793800, "time_this_iter_s": 225.5946009159088, "time_total_s": 30092.69255900383, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 30092.69255900383, "timesteps_since_restore": 7560000, "iterations_since_restore": 126, "perf": {"cpu_util_percent": 51.29316770186336, "ram_util_percent": 50.098757763975165}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3033.4231468098355, "episode_reward_min": 1321.103810498718, "episode_reward_mean": 2179.9617275406704, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.392951297448699, "mean_processing_ms": 1.4926434263958868, "mean_inference_ms": 0.8505888567590937}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7607808, "num_steps_sampled": 7620000, "sample_time_ms": 222488.67, "load_time_ms": 10.277, "grad_time_ms": 5025.364, "update_time_ms": 4.657, "learner": {"default_policy": {"cur_kl_coeff": 1.5407439784686627e-34, "cur_lr": 4.999999873689376e-05, "total_loss": 338.9652099609375, "policy_loss": 0.0004025684902444482, "vf_loss": 338.9648132324219, "vf_explained_var": -4.839693534819389e-09, "kl": 0.005759512074291706, "entropy": -1.2807590961456299, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7620000, "episodes_total": 2540, "training_iteration": 127, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-40-28", "timestamp": 1570794028, "time_this_iter_s": 227.34414887428284, "time_total_s": 30320.036707878113, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 30320.036707878113, "timesteps_since_restore": 7620000, "iterations_since_restore": 127, "perf": {"cpu_util_percent": 51.18055555555556, "ram_util_percent": 50.14382716049382}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3037.4853168816417, "episode_reward_min": 1321.103810498718, "episode_reward_mean": 2198.406143377301, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.390797719972908, "mean_processing_ms": 1.4920736276528817, "mean_inference_ms": 0.8501934479448069}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7667712, "num_steps_sampled": 7680000, "sample_time_ms": 222617.042, "load_time_ms": 10.156, "grad_time_ms": 5026.938, "update_time_ms": 4.622, "learner": {"default_policy": {"cur_kl_coeff": 7.703719892343314e-35, "cur_lr": 4.999999873689376e-05, "total_loss": 352.56549072265625, "policy_loss": 0.00034955545561388135, "vf_loss": 352.56512451171875, "vf_explained_var": -1.0952990514567773e-08, "kl": 0.004894816316664219, "entropy": -1.2931183576583862, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7680000, "episodes_total": 2560, "training_iteration": 128, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-44-16", "timestamp": 1570794256, "time_this_iter_s": 228.3664309978485, "time_total_s": 30548.40313887596, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 30548.40313887596, "timesteps_since_restore": 7680000, "iterations_since_restore": 128, "perf": {"cpu_util_percent": 51.4021472392638, "ram_util_percent": 50.18742331288343}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3037.4853168816417, "episode_reward_min": 1476.5492618048222, "episode_reward_mean": 2242.1038809597444, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.388764883258537, "mean_processing_ms": 1.4915607167820297, "mean_inference_ms": 0.8498014428754362}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7727616, "num_steps_sampled": 7740000, "sample_time_ms": 222912.816, "load_time_ms": 10.193, "grad_time_ms": 5023.862, "update_time_ms": 4.646, "learner": {"default_policy": {"cur_kl_coeff": 3.851859946171657e-35, "cur_lr": 4.999999873689376e-05, "total_loss": 394.87640380859375, "policy_loss": 2.344856875424739e-05, "vf_loss": 394.8763732910156, "vf_explained_var": -3.693450212338689e-09, "kl": 0.0066440817900002, "entropy": -1.331769347190857, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7740000, "episodes_total": 2580, "training_iteration": 129, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-48-06", "timestamp": 1570794486, "time_this_iter_s": 229.57926726341248, "time_total_s": 30777.982406139374, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 30777.982406139374, "timesteps_since_restore": 7740000, "iterations_since_restore": 129, "perf": {"cpu_util_percent": 51.64512195121951, "ram_util_percent": 50.200914634146336}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3037.4853168816417, "episode_reward_min": 1476.5492618048222, "episode_reward_mean": 2230.8651329685163, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.38676642905989, "mean_processing_ms": 1.4910501694570497, "mean_inference_ms": 0.8494238213301806}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7787520, "num_steps_sampled": 7800000, "sample_time_ms": 222731.956, "load_time_ms": 10.214, "grad_time_ms": 5020.702, "update_time_ms": 4.659, "learner": {"default_policy": {"cur_kl_coeff": 1.9259299730858284e-35, "cur_lr": 4.999999873689376e-05, "total_loss": 331.77386474609375, "policy_loss": 0.00028852379182353616, "vf_loss": 331.7735595703125, "vf_explained_var": -8.660504313695583e-09, "kl": 0.0059051718562841415, "entropy": -1.3130464553833008, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7800000, "episodes_total": 2600, "training_iteration": 130, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-51-53", "timestamp": 1570794713, "time_this_iter_s": 226.72895216941833, "time_total_s": 31004.711358308792, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 31004.711358308792, "timesteps_since_restore": 7800000, "iterations_since_restore": 130, "perf": {"cpu_util_percent": 51.349535603715175, "ram_util_percent": 50.1938080495356}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3037.4853168816417, "episode_reward_min": 1476.5492618048222, "episode_reward_mean": 2238.2221261098075, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.384849433361234, "mean_processing_ms": 1.4905851723290888, "mean_inference_ms": 0.8490553262881387}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7847424, "num_steps_sampled": 7860000, "sample_time_ms": 222300.144, "load_time_ms": 10.157, "grad_time_ms": 5025.665, "update_time_ms": 4.711, "learner": {"default_policy": {"cur_kl_coeff": 9.629649865429142e-36, "cur_lr": 4.999999873689376e-05, "total_loss": 333.77398681640625, "policy_loss": 2.017731821979396e-05, "vf_loss": 333.77398681640625, "vf_explained_var": -4.0755314678619925e-09, "kl": 0.00595882348716259, "entropy": -1.324370265007019, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7860000, "episodes_total": 2620, "training_iteration": 131, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-55-40", "timestamp": 1570794940, "time_this_iter_s": 227.29516005516052, "time_total_s": 31232.006518363953, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 31232.006518363953, "timesteps_since_restore": 7860000, "iterations_since_restore": 131, "perf": {"cpu_util_percent": 51.313580246913574, "ram_util_percent": 50.19814814814814}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3037.4853168816417, "episode_reward_min": 1315.4813396690183, "episode_reward_mean": 2198.8328680971335, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.382926391695807, "mean_processing_ms": 1.4900840726120073, "mean_inference_ms": 0.848692482690214}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7907328, "num_steps_sampled": 7920000, "sample_time_ms": 221856.045, "load_time_ms": 10.101, "grad_time_ms": 5028.389, "update_time_ms": 4.699, "learner": {"default_policy": {"cur_kl_coeff": 4.814824932714571e-36, "cur_lr": 4.999999873689376e-05, "total_loss": 284.7213439941406, "policy_loss": 0.0001491272560087964, "vf_loss": 284.7212219238281, "vf_explained_var": -4.202891812354892e-09, "kl": 0.005539558827877045, "entropy": -1.2887325286865234, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7920000, "episodes_total": 2640, "training_iteration": 132, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_04-59-25", "timestamp": 1570795165, "time_this_iter_s": 224.9325294494629, "time_total_s": 31456.939047813416, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 31456.939047813416, "timesteps_since_restore": 7920000, "iterations_since_restore": 132, "perf": {"cpu_util_percent": 51.35794392523365, "ram_util_percent": 50.19968847352025}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2914.4994178660495, "episode_reward_min": 1315.4813396690183, "episode_reward_mean": 2210.8419579488987, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.381025311447718, "mean_processing_ms": 1.4895990205230913, "mean_inference_ms": 0.8483285461262264}, "off_policy_estimator": {}, "info": {"num_steps_trained": 7967232, "num_steps_sampled": 7980000, "sample_time_ms": 222093.16, "load_time_ms": 10.152, "grad_time_ms": 5029.651, "update_time_ms": 4.685, "learner": {"default_policy": {"cur_kl_coeff": 2.4074124663572855e-36, "cur_lr": 4.999999873689376e-05, "total_loss": 374.6876525878906, "policy_loss": -0.00011911516776308417, "vf_loss": 374.68780517578125, "vf_explained_var": -1.1462431892539371e-08, "kl": 0.006542982999235392, "entropy": -1.3549070358276367, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 7980000, "episodes_total": 2660, "training_iteration": 133, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-03-13", "timestamp": 1570795393, "time_this_iter_s": 228.15981030464172, "time_total_s": 31685.098858118057, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 31685.098858118057, "timesteps_since_restore": 7980000, "iterations_since_restore": 133, "perf": {"cpu_util_percent": 51.30490797546012, "ram_util_percent": 50.20644171779141}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2914.4994178660495, "episode_reward_min": 1315.4813396690183, "episode_reward_mean": 2147.2172593307414, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.379007892396017, "mean_processing_ms": 1.4890622643743392, "mean_inference_ms": 0.847965638016519}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8027136, "num_steps_sampled": 8040000, "sample_time_ms": 221986.763, "load_time_ms": 10.165, "grad_time_ms": 5031.25, "update_time_ms": 4.716, "learner": {"default_policy": {"cur_kl_coeff": 1.2037062331786428e-36, "cur_lr": 4.999999873689376e-05, "total_loss": 299.0262145996094, "policy_loss": 0.0002473544445820153, "vf_loss": 299.02593994140625, "vf_explained_var": -7.896342246738186e-09, "kl": 0.004999789409339428, "entropy": -1.3221986293792725, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8040000, "episodes_total": 2680, "training_iteration": 134, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-06-59", "timestamp": 1570795619, "time_this_iter_s": 226.0365846157074, "time_total_s": 31911.135442733765, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 31911.135442733765, "timesteps_since_restore": 8040000, "iterations_since_restore": 134, "perf": {"cpu_util_percent": 51.468322981366455, "ram_util_percent": 50.292236024844726}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2914.4994178660495, "episode_reward_min": 1299.1913996940846, "episode_reward_mean": 2097.3217711064494, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.376944275509544, "mean_processing_ms": 1.4885588587970249, "mean_inference_ms": 0.8476114081224412}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8087040, "num_steps_sampled": 8100000, "sample_time_ms": 221983.904, "load_time_ms": 10.156, "grad_time_ms": 5027.588, "update_time_ms": 4.67, "learner": {"default_policy": {"cur_kl_coeff": 6.018531165893214e-37, "cur_lr": 4.999999873689376e-05, "total_loss": 266.26568603515625, "policy_loss": 0.00022065282973926514, "vf_loss": 266.2654724121094, "vf_explained_var": -4.967053879312289e-09, "kl": 0.004009648226201534, "entropy": -1.2973803281784058, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8100000, "episodes_total": 2700, "training_iteration": 135, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-10-45", "timestamp": 1570795845, "time_this_iter_s": 226.36364698410034, "time_total_s": 32137.499089717865, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 32137.499089717865, "timesteps_since_restore": 8100000, "iterations_since_restore": 135, "perf": {"cpu_util_percent": 51.24705882352942, "ram_util_percent": 50.29938080495357}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3004.663396526677, "episode_reward_min": 1299.1913996940846, "episode_reward_mean": 2125.072402909051, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.3749733216627895, "mean_processing_ms": 1.488075023046842, "mean_inference_ms": 0.8472679770651395}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8146944, "num_steps_sampled": 8160000, "sample_time_ms": 222287.343, "load_time_ms": 10.275, "grad_time_ms": 5229.981, "update_time_ms": 4.604, "learner": {"default_policy": {"cur_kl_coeff": 3.009265582946607e-37, "cur_lr": 4.999999873689376e-05, "total_loss": 378.9477844238281, "policy_loss": 0.0001447344693588093, "vf_loss": 378.9476623535156, "vf_explained_var": -5.603855601776786e-09, "kl": 0.006529606878757477, "entropy": -1.3991847038269043, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8160000, "episodes_total": 2720, "training_iteration": 136, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-14-36", "timestamp": 1570796076, "time_this_iter_s": 230.6580309867859, "time_total_s": 32368.15712070465, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 32368.15712070465, "timesteps_since_restore": 8160000, "iterations_since_restore": 136, "perf": {"cpu_util_percent": 52.170212765957444, "ram_util_percent": 50.35410334346505}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3004.663396526677, "episode_reward_min": 1244.613686736468, "episode_reward_mean": 2163.625640905617, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.373340340828945, "mean_processing_ms": 1.4876673651150072, "mean_inference_ms": 0.8469675405538404}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8206848, "num_steps_sampled": 8220000, "sample_time_ms": 222816.089, "load_time_ms": 10.372, "grad_time_ms": 5239.512, "update_time_ms": 4.575, "learner": {"default_policy": {"cur_kl_coeff": 1.5046327914733034e-37, "cur_lr": 4.999999873689376e-05, "total_loss": 346.3367004394531, "policy_loss": 0.0006198254995979369, "vf_loss": 346.3360595703125, "vf_explained_var": -7.386900424677378e-09, "kl": 0.005078369751572609, "entropy": -1.3735880851745605, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8220000, "episodes_total": 2740, "training_iteration": 137, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-18-29", "timestamp": 1570796309, "time_this_iter_s": 232.728036403656, "time_total_s": 32600.885157108307, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 32600.885157108307, "timesteps_since_restore": 8220000, "iterations_since_restore": 137, "perf": {"cpu_util_percent": 53.87289156626507, "ram_util_percent": 51.14397590361447}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3015.6982756844022, "episode_reward_min": 1244.613686736468, "episode_reward_mean": 2146.885572988166, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.371757589051944, "mean_processing_ms": 1.4872763627044572, "mean_inference_ms": 0.846683041768772}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8266752, "num_steps_sampled": 8280000, "sample_time_ms": 222797.616, "load_time_ms": 10.327, "grad_time_ms": 5245.298, "update_time_ms": 4.607, "learner": {"default_policy": {"cur_kl_coeff": 7.523163957366517e-38, "cur_lr": 4.999999873689376e-05, "total_loss": 348.5110168457031, "policy_loss": -3.231132041037199e-06, "vf_loss": 348.5110168457031, "vf_explained_var": -2.9292881453812925e-09, "kl": 0.006246337667107582, "entropy": -1.3953640460968018, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8280000, "episodes_total": 2760, "training_iteration": 138, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-22-17", "timestamp": 1570796537, "time_this_iter_s": 228.23901748657227, "time_total_s": 32829.12417459488, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 32829.12417459488, "timesteps_since_restore": 8280000, "iterations_since_restore": 138, "perf": {"cpu_util_percent": 51.53588957055215, "ram_util_percent": 51.46441717791413}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3015.6982756844022, "episode_reward_min": 1244.613686736468, "episode_reward_mean": 2186.0758802866053, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.370249880649552, "mean_processing_ms": 1.4868848406184705, "mean_inference_ms": 0.8464006353574385}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8326656, "num_steps_sampled": 8340000, "sample_time_ms": 222542.751, "load_time_ms": 10.331, "grad_time_ms": 5256.068, "update_time_ms": 4.599, "learner": {"default_policy": {"cur_kl_coeff": 3.7615819786832586e-38, "cur_lr": 4.999999873689376e-05, "total_loss": 358.5067443847656, "policy_loss": -0.00015637304750271142, "vf_loss": 358.5068664550781, "vf_explained_var": -2.5472071674137453e-10, "kl": 0.005731170065701008, "entropy": -1.414859652519226, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8340000, "episodes_total": 2780, "training_iteration": 139, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-26-04", "timestamp": 1570796764, "time_this_iter_s": 227.13796734809875, "time_total_s": 33056.26214194298, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 33056.26214194298, "timesteps_since_restore": 8340000, "iterations_since_restore": 139, "perf": {"cpu_util_percent": 51.298456790123446, "ram_util_percent": 50.5993827160494}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3015.6982756844022, "episode_reward_min": 1244.613686736468, "episode_reward_mean": 2260.8548271805503, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.368877929898831, "mean_processing_ms": 1.4864768328432278, "mean_inference_ms": 0.8461183979887643}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8386560, "num_steps_sampled": 8400000, "sample_time_ms": 222580.937, "load_time_ms": 10.301, "grad_time_ms": 5260.938, "update_time_ms": 4.604, "learner": {"default_policy": {"cur_kl_coeff": 1.8807909893416293e-38, "cur_lr": 4.999999873689376e-05, "total_loss": 368.90045166015625, "policy_loss": 0.0002921150007750839, "vf_loss": 368.9001159667969, "vf_explained_var": -1.0188828669654981e-09, "kl": 0.00673392228782177, "entropy": -1.43276047706604, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8400000, "episodes_total": 2800, "training_iteration": 140, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-29-51", "timestamp": 1570796991, "time_this_iter_s": 227.15953612327576, "time_total_s": 33283.421678066254, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 33283.421678066254, "timesteps_since_restore": 8400000, "iterations_since_restore": 140, "perf": {"cpu_util_percent": 51.38395061728396, "ram_util_percent": 50.63796296296295}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3015.6982756844022, "episode_reward_min": 1244.613686736468, "episode_reward_mean": 2249.04411014296, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.367493266384629, "mean_processing_ms": 1.4860623085120963, "mean_inference_ms": 0.8458380023414043}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8446464, "num_steps_sampled": 8460000, "sample_time_ms": 222650.211, "load_time_ms": 10.307, "grad_time_ms": 5268.94, "update_time_ms": 4.657, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 355.1369323730469, "policy_loss": 7.804362394381315e-05, "vf_loss": 355.1368103027344, "vf_explained_var": -3.4387295233528903e-09, "kl": 0.006698769051581621, "entropy": -1.439775824546814, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8460000, "episodes_total": 2820, "training_iteration": 141, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-33-39", "timestamp": 1570797219, "time_this_iter_s": 228.06814575195312, "time_total_s": 33511.48982381821, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 33511.48982381821, "timesteps_since_restore": 8460000, "iterations_since_restore": 141, "perf": {"cpu_util_percent": 51.26123076923077, "ram_util_percent": 50.695692307692305}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3015.6982756844022, "episode_reward_min": 1270.7331095516838, "episode_reward_mean": 2243.260530907779, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.365890332094778, "mean_processing_ms": 1.4855820354388183, "mean_inference_ms": 0.8455254971750367}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8506368, "num_steps_sampled": 8520000, "sample_time_ms": 222765.244, "load_time_ms": 10.303, "grad_time_ms": 5272.515, "update_time_ms": 4.686, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 327.8664855957031, "policy_loss": 0.0005435719504021108, "vf_loss": 327.86590576171875, "vf_explained_var": -5.858576290762585e-09, "kl": 0.0040824743919074535, "entropy": -1.4202516078948975, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8520000, "episodes_total": 2840, "training_iteration": 142, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-37-26", "timestamp": 1570797446, "time_this_iter_s": 226.11893606185913, "time_total_s": 33737.608759880066, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 33737.608759880066, "timesteps_since_restore": 8520000, "iterations_since_restore": 142, "perf": {"cpu_util_percent": 51.33034055727554, "ram_util_percent": 50.699999999999996}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3008.4574808206835, "episode_reward_min": 1515.5315443793047, "episode_reward_mean": 2245.5672120196114, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.364274087518831, "mean_processing_ms": 1.4851106844626492, "mean_inference_ms": 0.8452065279797571}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8566272, "num_steps_sampled": 8580000, "sample_time_ms": 222676.693, "load_time_ms": 10.262, "grad_time_ms": 5277.496, "update_time_ms": 4.669, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 348.1716613769531, "policy_loss": -0.000261071661952883, "vf_loss": 348.17193603515625, "vf_explained_var": -1.528324244937096e-09, "kl": 0.006652958691120148, "entropy": -1.4362887144088745, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8580000, "episodes_total": 2860, "training_iteration": 143, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-41-13", "timestamp": 1570797673, "time_this_iter_s": 227.32266759872437, "time_total_s": 33964.93142747879, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 33964.93142747879, "timesteps_since_restore": 8580000, "iterations_since_restore": 143, "perf": {"cpu_util_percent": 51.56481481481482, "ram_util_percent": 50.69938271604938}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2991.6470809203524, "episode_reward_min": 1369.3448273987433, "episode_reward_mean": 2207.9572412199586, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.362645457430816, "mean_processing_ms": 1.4847065200354976, "mean_inference_ms": 0.8448974375852236}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8626176, "num_steps_sampled": 8640000, "sample_time_ms": 222771.126, "load_time_ms": 10.226, "grad_time_ms": 5283.586, "update_time_ms": 4.642, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 300.0270080566406, "policy_loss": -6.22580701019615e-05, "vf_loss": 300.0270690917969, "vf_explained_var": -7.514261213259488e-09, "kl": 0.0059175798669457436, "entropy": -1.4059268236160278, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8640000, "episodes_total": 2880, "training_iteration": 144, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-45-00", "timestamp": 1570797900, "time_this_iter_s": 227.04129576683044, "time_total_s": 34191.97272324562, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 34191.97272324562, "timesteps_since_restore": 8640000, "iterations_since_restore": 144, "perf": {"cpu_util_percent": 51.69012345679013, "ram_util_percent": 50.70987654320987}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2991.6470809203524, "episode_reward_min": 1358.8131262128595, "episode_reward_mean": 2179.6720741989375, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.360991480309143, "mean_processing_ms": 1.4843293010526113, "mean_inference_ms": 0.8445919415356835}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8686080, "num_steps_sampled": 8700000, "sample_time_ms": 222825.042, "load_time_ms": 10.273, "grad_time_ms": 5290.188, "update_time_ms": 4.638, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 333.3490905761719, "policy_loss": 0.00036329388967715204, "vf_loss": 333.3487548828125, "vf_explained_var": -1.2608674992975466e-08, "kl": 0.005710863973945379, "entropy": -1.4260759353637695, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8700000, "episodes_total": 2900, "training_iteration": 145, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-48-47", "timestamp": 1570798127, "time_this_iter_s": 226.96924018859863, "time_total_s": 34418.94196343422, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 34418.94196343422, "timesteps_since_restore": 8700000, "iterations_since_restore": 145, "perf": {"cpu_util_percent": 51.28641975308642, "ram_util_percent": 50.72716049382716}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 2991.6470809203524, "episode_reward_min": 1358.8131262128595, "episode_reward_mean": 2159.994130040973, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.359318093317771, "mean_processing_ms": 1.483982251462035, "mean_inference_ms": 0.8442901115009624}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8745984, "num_steps_sampled": 8760000, "sample_time_ms": 222775.209, "load_time_ms": 10.11, "grad_time_ms": 5089.162, "update_time_ms": 4.568, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 330.0295715332031, "policy_loss": -0.00018976615683641285, "vf_loss": 330.02978515625, "vf_explained_var": -5.349134912790987e-09, "kl": 0.007420304697006941, "entropy": -1.4448356628417969, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8760000, "episodes_total": 2920, "training_iteration": 146, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-52-35", "timestamp": 1570798355, "time_this_iter_s": 228.14227557182312, "time_total_s": 34647.08423900604, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 34647.08423900604, "timesteps_since_restore": 8760000, "iterations_since_restore": 146, "perf": {"cpu_util_percent": 51.09600000000001, "ram_util_percent": 50.79538461538461}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3029.8487228170975, "episode_reward_min": 1358.8131262128595, "episode_reward_mean": 2187.1887592519556, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.357695230678243, "mean_processing_ms": 1.4836457334861972, "mean_inference_ms": 0.8439900268585293}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8805888, "num_steps_sampled": 8820000, "sample_time_ms": 222187.694, "load_time_ms": 9.992, "grad_time_ms": 5087.422, "update_time_ms": 4.52, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 373.81109619140625, "policy_loss": 0.0008763225050643086, "vf_loss": 373.8102111816406, "vf_explained_var": -7.1321797356915795e-09, "kl": 0.007244299631565809, "entropy": -1.4906598329544067, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8820000, "episodes_total": 2940, "training_iteration": 147, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_05-56-22", "timestamp": 1570798582, "time_this_iter_s": 226.83327293395996, "time_total_s": 34873.91751194, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 34873.91751194, "timesteps_since_restore": 8820000, "iterations_since_restore": 147, "perf": {"cpu_util_percent": 51.35864197530864, "ram_util_percent": 50.79907407407408}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3029.8487228170975, "episode_reward_min": 1358.8131262128595, "episode_reward_mean": 2218.2401870087388, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.356159607516052, "mean_processing_ms": 1.4832842715085084, "mean_inference_ms": 0.8437037015267052}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8865792, "num_steps_sampled": 8880000, "sample_time_ms": 222271.767, "load_time_ms": 10.051, "grad_time_ms": 5085.31, "update_time_ms": 4.454, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 393.29339599609375, "policy_loss": 0.000779160181991756, "vf_loss": 393.2926025390625, "vf_explained_var": -3.5660898678457897e-09, "kl": 0.007883390411734581, "entropy": -1.523728847503662, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8880000, "episodes_total": 2960, "training_iteration": 148, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-00-11", "timestamp": 1570798811, "time_this_iter_s": 229.05873894691467, "time_total_s": 35102.97625088692, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 35102.97625088692, "timesteps_since_restore": 8880000, "iterations_since_restore": 148, "perf": {"cpu_util_percent": 51.36697247706422, "ram_util_percent": 50.80061162079512}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3057.5341493160295, "episode_reward_min": 1358.8131262128595, "episode_reward_mean": 2230.671498066657, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.354651983044246, "mean_processing_ms": 1.482876203436275, "mean_inference_ms": 0.8434208871911432}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8925696, "num_steps_sampled": 8940000, "sample_time_ms": 222224.385, "load_time_ms": 10.001, "grad_time_ms": 5081.766, "update_time_ms": 4.43, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 315.65411376953125, "policy_loss": 0.0003467268543317914, "vf_loss": 315.6537780761719, "vf_explained_var": 1.2736035837068727e-10, "kl": 0.007142008747905493, "entropy": -1.4739017486572266, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 8940000, "episodes_total": 2980, "training_iteration": 149, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-03-58", "timestamp": 1570799038, "time_this_iter_s": 226.6279480457306, "time_total_s": 35329.60419893265, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 35329.60419893265, "timesteps_since_restore": 8940000, "iterations_since_restore": 149, "perf": {"cpu_util_percent": 51.369969040247675, "ram_util_percent": 50.86470588235293}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3057.5341493160295, "episode_reward_min": 1474.7052670853525, "episode_reward_mean": 2214.1094697920144, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.353170612461417, "mean_processing_ms": 1.4824547609971188, "mean_inference_ms": 0.8431499035155278}, "off_policy_estimator": {}, "info": {"num_steps_trained": 8985600, "num_steps_sampled": 9000000, "sample_time_ms": 222178.792, "load_time_ms": 10.04, "grad_time_ms": 5081.559, "update_time_ms": 4.378, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 306.48236083984375, "policy_loss": 0.000351724709616974, "vf_loss": 306.48199462890625, "vf_explained_var": -5.349134912790987e-09, "kl": 0.0059328218922019005, "entropy": -1.4586695432662964, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9000000, "episodes_total": 3000, "training_iteration": 150, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-07-44", "timestamp": 1570799264, "time_this_iter_s": 226.70096349716187, "time_total_s": 35556.30516242981, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 35556.30516242981, "timesteps_since_restore": 9000000, "iterations_since_restore": 150, "perf": {"cpu_util_percent": 51.32074303405573, "ram_util_percent": 50.90092879256965}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3057.5341493160295, "episode_reward_min": 1474.7052670853525, "episode_reward_mean": 2224.105571036483, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.3517558760098, "mean_processing_ms": 1.4820418964062947, "mean_inference_ms": 0.8428896014343025}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9045504, "num_steps_sampled": 9060000, "sample_time_ms": 222162.545, "load_time_ms": 10.816, "grad_time_ms": 5078.772, "update_time_ms": 4.303, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 344.5024719238281, "policy_loss": -0.0006586529780179262, "vf_loss": 344.503173828125, "vf_explained_var": -7.514261213259488e-09, "kl": 0.007243610452860594, "entropy": -1.5162338018417358, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9060000, "episodes_total": 3020, "training_iteration": 151, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-11-32", "timestamp": 1570799492, "time_this_iter_s": 227.88481163978577, "time_total_s": 35784.189974069595, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 35784.189974069595, "timesteps_since_restore": 9060000, "iterations_since_restore": 151, "perf": {"cpu_util_percent": 51.54769230769231, "ram_util_percent": 50.91107692307692}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3057.5341493160295, "episode_reward_min": 1474.7052670853525, "episode_reward_mean": 2224.1196414428223, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.350540867258457, "mean_processing_ms": 1.481759550314748, "mean_inference_ms": 0.8426615768918236}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9105408, "num_steps_sampled": 9120000, "sample_time_ms": 222929.011, "load_time_ms": 10.851, "grad_time_ms": 5078.523, "update_time_ms": 4.228, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 368.3914794921875, "policy_loss": 0.0002147915802197531, "vf_loss": 368.39129638671875, "vf_explained_var": -6.368017668734183e-09, "kl": 0.007881799712777138, "entropy": -1.565125823020935, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9120000, "episodes_total": 3040, "training_iteration": 152, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-15-26", "timestamp": 1570799726, "time_this_iter_s": 233.78085684776306, "time_total_s": 36017.97083091736, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 36017.97083091736, "timesteps_since_restore": 9120000, "iterations_since_restore": 152, "perf": {"cpu_util_percent": 54.374251497006, "ram_util_percent": 51.065568862275455}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3057.5341493160295, "episode_reward_min": 1474.7052670853525, "episode_reward_mean": 2218.8924527032464, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.349301529049694, "mean_processing_ms": 1.4815420060826594, "mean_inference_ms": 0.8424302987175157}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9165312, "num_steps_sampled": 9180000, "sample_time_ms": 223194.821, "load_time_ms": 10.912, "grad_time_ms": 5078.04, "update_time_ms": 4.194, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 388.4996032714844, "policy_loss": 0.00033031264320015907, "vf_loss": 388.499267578125, "vf_explained_var": -8.533143969202683e-09, "kl": 0.0042679812759160995, "entropy": -1.571805477142334, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9180000, "episodes_total": 3060, "training_iteration": 153, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-19-16", "timestamp": 1570799956, "time_this_iter_s": 229.9773190021515, "time_total_s": 36247.94814991951, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 36247.94814991951, "timesteps_since_restore": 9180000, "iterations_since_restore": 153, "perf": {"cpu_util_percent": 51.66310975609756, "ram_util_percent": 52.53079268292682}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3064.9934679367625, "episode_reward_min": 1474.7052670853525, "episode_reward_mean": 2289.7909737898376, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.348167991504749, "mean_processing_ms": 1.4813427821833463, "mean_inference_ms": 0.8421961823977767}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9225216, "num_steps_sampled": 9240000, "sample_time_ms": 223359.128, "load_time_ms": 10.937, "grad_time_ms": 5079.546, "update_time_ms": 4.096, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 433.9513854980469, "policy_loss": 0.0006739410455338657, "vf_loss": 433.9507141113281, "vf_explained_var": -8.533143969202683e-09, "kl": 0.00816893670707941, "entropy": -1.6056526899337769, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9240000, "episodes_total": 3080, "training_iteration": 154, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-23-05", "timestamp": 1570800185, "time_this_iter_s": 228.6989300251007, "time_total_s": 36476.64707994461, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 36476.64707994461, "timesteps_since_restore": 9240000, "iterations_since_restore": 154, "perf": {"cpu_util_percent": 51.42607361963191, "ram_util_percent": 55.227607361963194}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3064.9934679367625, "episode_reward_min": 1353.556173376261, "episode_reward_mean": 2298.5631719351873, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.347071149900094, "mean_processing_ms": 1.4811919368471413, "mean_inference_ms": 0.8419666703778339}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9285120, "num_steps_sampled": 9300000, "sample_time_ms": 223389.762, "load_time_ms": 10.915, "grad_time_ms": 5079.215, "update_time_ms": 4.082, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 319.53692626953125, "policy_loss": -3.49509937223047e-05, "vf_loss": 319.5369873046875, "vf_explained_var": -2.801927800888393e-09, "kl": 0.006182338111102581, "entropy": -1.5216206312179565, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9300000, "episodes_total": 3100, "training_iteration": 155, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-26-52", "timestamp": 1570800412, "time_this_iter_s": 227.27215909957886, "time_total_s": 36703.91923904419, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 36703.91923904419, "timesteps_since_restore": 9300000, "iterations_since_restore": 155, "perf": {"cpu_util_percent": 51.626543209876544, "ram_util_percent": 55.300000000000004}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3064.9934679367625, "episode_reward_min": 1353.556173376261, "episode_reward_mean": 2322.818341828732, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.346011040367947, "mean_processing_ms": 1.481033668917287, "mean_inference_ms": 0.8417336354604362}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9345024, "num_steps_sampled": 9360000, "sample_time_ms": 223378.541, "load_time_ms": 10.963, "grad_time_ms": 5080.387, "update_time_ms": 4.213, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 390.2936706542969, "policy_loss": 0.0007284220191650093, "vf_loss": 390.2929382324219, "vf_explained_var": -3.5660898678457897e-09, "kl": 0.0063081602565944195, "entropy": -1.5931533575057983, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9360000, "episodes_total": 3120, "training_iteration": 156, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-30-40", "timestamp": 1570800640, "time_this_iter_s": 228.04396176338196, "time_total_s": 36931.96320080757, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 36931.96320080757, "timesteps_since_restore": 9360000, "iterations_since_restore": 156, "perf": {"cpu_util_percent": 51.50337423312883, "ram_util_percent": 55.3708588957055}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3064.9934679367625, "episode_reward_min": 1353.556173376261, "episode_reward_mean": 2306.32606831232, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.344809688630807, "mean_processing_ms": 1.4808146650910892, "mean_inference_ms": 0.8414819457011049}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9404928, "num_steps_sampled": 9420000, "sample_time_ms": 223528.995, "load_time_ms": 11.014, "grad_time_ms": 5081.321, "update_time_ms": 4.238, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 350.3853454589844, "policy_loss": 0.0009383868891745806, "vf_loss": 350.3844299316406, "vf_explained_var": -1.1462432114583976e-09, "kl": 0.0073346407152712345, "entropy": -1.5803309679031372, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9420000, "episodes_total": 3140, "training_iteration": 157, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-34-28", "timestamp": 1570800868, "time_this_iter_s": 228.34850311279297, "time_total_s": 37160.311703920364, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 37160.311703920364, "timesteps_since_restore": 9420000, "iterations_since_restore": 157, "perf": {"cpu_util_percent": 51.57692307692308, "ram_util_percent": 55.39876923076922}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3064.9934679367625, "episode_reward_min": 1353.556173376261, "episode_reward_mean": 2281.101465814145, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.343573569809669, "mean_processing_ms": 1.4805231320090535, "mean_inference_ms": 0.8412314575469421}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9464832, "num_steps_sampled": 9480000, "sample_time_ms": 223243.398, "load_time_ms": 10.972, "grad_time_ms": 5083.566, "update_time_ms": 4.301, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 354.1051025390625, "policy_loss": -0.0002704828802961856, "vf_loss": 354.10540771484375, "vf_explained_var": -1.171715258152517e-08, "kl": 0.008633880876004696, "entropy": -1.6004753112792969, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9480000, "episodes_total": 3160, "training_iteration": 158, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-38-15", "timestamp": 1570801095, "time_this_iter_s": 226.2251172065735, "time_total_s": 37386.53682112694, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 37386.53682112694, "timesteps_since_restore": 9480000, "iterations_since_restore": 158, "perf": {"cpu_util_percent": 51.342414860681124, "ram_util_percent": 55.42724458204334}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3029.004997205949, "episode_reward_min": 1353.556173376261, "episode_reward_mean": 2235.2487572029268, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.342278227971353, "mean_processing_ms": 1.4802304951242053, "mean_inference_ms": 0.8409836074254764}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9524736, "num_steps_sampled": 9540000, "sample_time_ms": 223312.729, "load_time_ms": 11.003, "grad_time_ms": 5086.304, "update_time_ms": 4.303, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 358.5614929199219, "policy_loss": 0.00045807138667441905, "vf_loss": 358.56103515625, "vf_explained_var": -9.806747414131678e-09, "kl": 0.006831216160207987, "entropy": -1.613525629043579, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9540000, "episodes_total": 3180, "training_iteration": 159, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-42-02", "timestamp": 1570801322, "time_this_iter_s": 227.34941363334656, "time_total_s": 37613.886234760284, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 37613.886234760284, "timesteps_since_restore": 9540000, "iterations_since_restore": 159, "perf": {"cpu_util_percent": 51.23446153846154, "ram_util_percent": 55.49938461538461}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3042.2847514446726, "episode_reward_min": 1451.9859092532047, "episode_reward_mean": 2276.989349366336, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.34102737516111, "mean_processing_ms": 1.4799183064437402, "mean_inference_ms": 0.8407323873912261}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9584640, "num_steps_sampled": 9600000, "sample_time_ms": 223496.828, "load_time_ms": 10.953, "grad_time_ms": 5085.68, "update_time_ms": 4.35, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 381.3824462890625, "policy_loss": 0.0008727480308152735, "vf_loss": 381.3816223144531, "vf_explained_var": -1.528324244937096e-09, "kl": 0.0076065571047365665, "entropy": -1.653287410736084, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9600000, "episodes_total": 3200, "training_iteration": 160, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-45-51", "timestamp": 1570801551, "time_this_iter_s": 228.53584671020508, "time_total_s": 37842.42208147049, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 37842.42208147049, "timesteps_since_restore": 9600000, "iterations_since_restore": 160, "perf": {"cpu_util_percent": 51.33558282208588, "ram_util_percent": 55.37208588957056}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3042.2847514446726, "episode_reward_min": 1451.9859092532047, "episode_reward_mean": 2271.415351346707, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.339839717209313, "mean_processing_ms": 1.4796208563731312, "mean_inference_ms": 0.8404962201768873}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9644544, "num_steps_sampled": 9660000, "sample_time_ms": 223685.657, "load_time_ms": 10.262, "grad_time_ms": 5086.67, "update_time_ms": 4.402, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 376.97265625, "policy_loss": 0.0006709249573759735, "vf_loss": 376.9719543457031, "vf_explained_var": -5.603855601776786e-09, "kl": 0.008511237800121307, "entropy": -1.6616114377975464, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9660000, "episodes_total": 3220, "training_iteration": 161, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-49-40", "timestamp": 1570801780, "time_this_iter_s": 229.7768681049347, "time_total_s": 38072.198949575424, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 38072.198949575424, "timesteps_since_restore": 9660000, "iterations_since_restore": 161, "perf": {"cpu_util_percent": 52.079510703363916, "ram_util_percent": 55.34342507645258}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3096.7730127033965, "episode_reward_min": 1451.9859092532047, "episode_reward_mean": 2279.3526997793947, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.338644490268535, "mean_processing_ms": 1.4793028722546813, "mean_inference_ms": 0.8402626793614323}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9704448, "num_steps_sampled": 9720000, "sample_time_ms": 223101.862, "load_time_ms": 10.258, "grad_time_ms": 5087.585, "update_time_ms": 4.518, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 366.34429931640625, "policy_loss": 0.0012409072369337082, "vf_loss": 366.343017578125, "vf_explained_var": -7.259540080184479e-09, "kl": 0.00677101407200098, "entropy": -1.6516884565353394, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9720000, "episodes_total": 3240, "training_iteration": 162, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-53-28", "timestamp": 1570802008, "time_this_iter_s": 227.95323491096497, "time_total_s": 38300.15218448639, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 38300.15218448639, "timesteps_since_restore": 9720000, "iterations_since_restore": 162, "perf": {"cpu_util_percent": 51.306748466257666, "ram_util_percent": 55.3963190184049}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3096.7730127033965, "episode_reward_min": 1417.7181955880533, "episode_reward_mean": 2292.400165494767, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.337476606179142, "mean_processing_ms": 1.479024686978564, "mean_inference_ms": 0.8400321592535793}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9764352, "num_steps_sampled": 9780000, "sample_time_ms": 222907.432, "load_time_ms": 10.21, "grad_time_ms": 5087.195, "update_time_ms": 4.566, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 376.63421630859375, "policy_loss": 0.0018430943600833416, "vf_loss": 376.63238525390625, "vf_explained_var": -9.042585347174281e-09, "kl": 0.007729583885520697, "entropy": -1.672771692276001, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9780000, "episodes_total": 3260, "training_iteration": 163, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_06-57-16", "timestamp": 1570802236, "time_this_iter_s": 228.0283966064453, "time_total_s": 38528.180581092834, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 38528.180581092834, "timesteps_since_restore": 9780000, "iterations_since_restore": 163, "perf": {"cpu_util_percent": 51.25846153846154, "ram_util_percent": 55.42584615384614}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3096.7730127033965, "episode_reward_min": 1417.7181955880533, "episode_reward_mean": 2331.6083649641278, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.3363840540295975, "mean_processing_ms": 1.4787958578956728, "mean_inference_ms": 0.8398038808912065}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9824256, "num_steps_sampled": 9840000, "sample_time_ms": 223073.074, "load_time_ms": 10.174, "grad_time_ms": 5086.576, "update_time_ms": 4.623, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 421.5362548828125, "policy_loss": 0.0011985857272520661, "vf_loss": 421.5350646972656, "vf_explained_var": -7.386900424677378e-09, "kl": 0.006897140759974718, "entropy": -1.7264200448989868, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9840000, "episodes_total": 3280, "training_iteration": 164, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-01-07", "timestamp": 1570802467, "time_this_iter_s": 230.34946846961975, "time_total_s": 38758.530049562454, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 38758.530049562454, "timesteps_since_restore": 9840000, "iterations_since_restore": 164, "perf": {"cpu_util_percent": 51.208841463414636, "ram_util_percent": 55.4844512195122}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3096.7730127033965, "episode_reward_min": 1381.795146883838, "episode_reward_mean": 2322.2649012357447, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.335296025113192, "mean_processing_ms": 1.4785701696964457, "mean_inference_ms": 0.8395771600303431}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9884160, "num_steps_sampled": 9900000, "sample_time_ms": 223048.272, "load_time_ms": 10.151, "grad_time_ms": 5085.105, "update_time_ms": 4.592, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 370.12017822265625, "policy_loss": 0.0008794529130682349, "vf_loss": 370.11932373046875, "vf_explained_var": -7.514261213259488e-09, "kl": 0.007923475466668606, "entropy": -1.6857450008392334, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9900000, "episodes_total": 3300, "training_iteration": 165, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-04-54", "timestamp": 1570802694, "time_this_iter_s": 227.00873231887817, "time_total_s": 38985.53878188133, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 38985.53878188133, "timesteps_since_restore": 9900000, "iterations_since_restore": 165, "perf": {"cpu_util_percent": 51.53981481481482, "ram_util_percent": 55.498148148148154}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3096.7730127033965, "episode_reward_min": 1381.795146883838, "episode_reward_mean": 2324.3306171278286, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.334173849646387, "mean_processing_ms": 1.4783422650956652, "mean_inference_ms": 0.839349903255112}, "off_policy_estimator": {}, "info": {"num_steps_trained": 9944064, "num_steps_sampled": 9960000, "sample_time_ms": 223079.679, "load_time_ms": 10.169, "grad_time_ms": 5086.565, "update_time_ms": 4.607, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 377.6601867675781, "policy_loss": 0.002230962971225381, "vf_loss": 377.6579284667969, "vf_explained_var": -1.0188828669654981e-09, "kl": 0.00923401489853859, "entropy": -1.7261074781417847, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 9960000, "episodes_total": 3320, "training_iteration": 166, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-08-42", "timestamp": 1570802922, "time_this_iter_s": 228.37273335456848, "time_total_s": 39213.9115152359, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 39213.9115152359, "timesteps_since_restore": 9960000, "iterations_since_restore": 166, "perf": {"cpu_util_percent": 51.57392638036809, "ram_util_percent": 55.50828220858897}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3052.176600792995, "episode_reward_min": 1381.795146883838, "episode_reward_mean": 2357.590241327462, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.333127871085545, "mean_processing_ms": 1.4781177331046573, "mean_inference_ms": 0.8391280804940919}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10003968, "num_steps_sampled": 10020000, "sample_time_ms": 223203.96, "load_time_ms": 10.192, "grad_time_ms": 5082.857, "update_time_ms": 4.615, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 409.90966796875, "policy_loss": 0.0011881475802510977, "vf_loss": 409.908447265625, "vf_explained_var": -2.4198467674096946e-09, "kl": 0.00955733098089695, "entropy": -1.7725549936294556, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10020000, "episodes_total": 3340, "training_iteration": 167, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-12-32", "timestamp": 1570803152, "time_this_iter_s": 229.55401587486267, "time_total_s": 39443.46553111076, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 39443.46553111076, "timesteps_since_restore": 10020000, "iterations_since_restore": 167, "perf": {"cpu_util_percent": 51.340853658536595, "ram_util_percent": 55.59603658536587}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3052.176600792995, "episode_reward_min": 1381.795146883838, "episode_reward_mean": 2354.666773378418, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.332240199268739, "mean_processing_ms": 1.4779118357968475, "mean_inference_ms": 0.8389328324412136}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10063872, "num_steps_sampled": 10080000, "sample_time_ms": 223768.458, "load_time_ms": 10.234, "grad_time_ms": 5080.151, "update_time_ms": 4.531, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 369.3013610839844, "policy_loss": 0.0004047710681334138, "vf_loss": 369.3009338378906, "vf_explained_var": -8.787864658188482e-09, "kl": 0.008358349092304707, "entropy": -1.7276036739349365, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10080000, "episodes_total": 3360, "training_iteration": 168, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-16-23", "timestamp": 1570803383, "time_this_iter_s": 231.84287452697754, "time_total_s": 39675.30840563774, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 39675.30840563774, "timesteps_since_restore": 10080000, "iterations_since_restore": 168, "perf": {"cpu_util_percent": 54.032424242424234, "ram_util_percent": 55.720606060606066}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3052.176600792995, "episode_reward_min": 1381.795146883838, "episode_reward_mean": 2327.8122927546174, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.331351283884022, "mean_processing_ms": 1.4776509727492602, "mean_inference_ms": 0.8387472494796107}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10123776, "num_steps_sampled": 10140000, "sample_time_ms": 223834.355, "load_time_ms": 10.395, "grad_time_ms": 5078.363, "update_time_ms": 4.598, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 380.6027526855469, "policy_loss": 0.0008131205104291439, "vf_loss": 380.6019287109375, "vf_explained_var": -6.368017779756485e-10, "kl": 0.006627724505960941, "entropy": -1.7412049770355225, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10140000, "episodes_total": 3380, "training_iteration": 169, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-20-11", "timestamp": 1570803611, "time_this_iter_s": 227.99224424362183, "time_total_s": 39903.30064988136, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 39903.30064988136, "timesteps_since_restore": 10140000, "iterations_since_restore": 169, "perf": {"cpu_util_percent": 51.76779141104296, "ram_util_percent": 53.49233128834355}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3052.176600792995, "episode_reward_min": 1552.5969516358923, "episode_reward_mean": 2331.345813736235, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.330468666838864, "mean_processing_ms": 1.4773921399831624, "mean_inference_ms": 0.8385629159773883}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10183680, "num_steps_sampled": 10200000, "sample_time_ms": 223699.368, "load_time_ms": 10.394, "grad_time_ms": 5078.525, "update_time_ms": 4.653, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 382.2425231933594, "policy_loss": 0.0008789993589743972, "vf_loss": 382.24163818359375, "vf_explained_var": -3.948171123369093e-09, "kl": 0.007721083704382181, "entropy": -1.7556211948394775, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10200000, "episodes_total": 3400, "training_iteration": 170, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-23-59", "timestamp": 1570803839, "time_this_iter_s": 227.18774437904358, "time_total_s": 40130.48839426041, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 40130.48839426041, "timesteps_since_restore": 10200000, "iterations_since_restore": 170, "perf": {"cpu_util_percent": 51.51574074074074, "ram_util_percent": 51.41635802469135}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3041.4282666399654, "episode_reward_min": 1552.5969516358923, "episode_reward_mean": 2335.5679607765587, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.329592778609273, "mean_processing_ms": 1.4771153921602573, "mean_inference_ms": 0.8383746656545878}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10243584, "num_steps_sampled": 10260000, "sample_time_ms": 223542.754, "load_time_ms": 10.309, "grad_time_ms": 5075.766, "update_time_ms": 4.68, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 387.7888488769531, "policy_loss": 0.0009865574538707733, "vf_loss": 387.7878723144531, "vf_explained_var": -1.273603555951297e-09, "kl": 0.0071579329669475555, "entropy": -1.789628028869629, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10260000, "episodes_total": 3420, "training_iteration": 171, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-27-47", "timestamp": 1570804067, "time_this_iter_s": 228.18240404129028, "time_total_s": 40358.6707983017, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 40358.6707983017, "timesteps_since_restore": 10260000, "iterations_since_restore": 171, "perf": {"cpu_util_percent": 51.366769230769236, "ram_util_percent": 51.49292307692307}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3041.277396390285, "episode_reward_min": 1552.5969516358923, "episode_reward_mean": 2291.3922025148477, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.328647421407559, "mean_processing_ms": 1.4768448623325172, "mean_inference_ms": 0.8381834846618813}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10303488, "num_steps_sampled": 10320000, "sample_time_ms": 223531.833, "load_time_ms": 10.324, "grad_time_ms": 5075.787, "update_time_ms": 4.652, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 348.61578369140625, "policy_loss": 0.00047967577120289207, "vf_loss": 348.6153259277344, "vf_explained_var": -3.8208107788761936e-09, "kl": 0.009670348837971687, "entropy": -1.752366304397583, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10320000, "episodes_total": 3440, "training_iteration": 172, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-31-35", "timestamp": 1570804295, "time_this_iter_s": 227.84405541419983, "time_total_s": 40586.5148537159, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 40586.5148537159, "timesteps_since_restore": 10320000, "iterations_since_restore": 172, "perf": {"cpu_util_percent": 51.2883076923077, "ram_util_percent": 51.49230769230769}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3054.637216163588, "episode_reward_min": 1570.1410214240584, "episode_reward_mean": 2285.7422633450497, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.327610210989354, "mean_processing_ms": 1.4765507393155133, "mean_inference_ms": 0.8379799893952782}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10363392, "num_steps_sampled": 10380000, "sample_time_ms": 223596.012, "load_time_ms": 10.306, "grad_time_ms": 5071.499, "update_time_ms": 4.609, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 360.74285888671875, "policy_loss": 0.0010298284469172359, "vf_loss": 360.7418518066406, "vf_explained_var": -5.603855601776786e-09, "kl": 0.006654420401901007, "entropy": -1.7767667770385742, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10380000, "episodes_total": 3460, "training_iteration": 173, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-35-23", "timestamp": 1570804523, "time_this_iter_s": 228.62696480751038, "time_total_s": 40815.14181852341, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 40815.14181852341, "timesteps_since_restore": 10380000, "iterations_since_restore": 173, "perf": {"cpu_util_percent": 51.26901840490799, "ram_util_percent": 51.48312883435583}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3054.637216163588, "episode_reward_min": 1571.5596343299374, "episode_reward_mean": 2307.4846784830906, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.326640062500885, "mean_processing_ms": 1.4763131403663496, "mean_inference_ms": 0.8377827494028618}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10423296, "num_steps_sampled": 10440000, "sample_time_ms": 223599.487, "load_time_ms": 10.318, "grad_time_ms": 5069.426, "update_time_ms": 4.637, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 412.6548767089844, "policy_loss": 0.0013235356891527772, "vf_loss": 412.653564453125, "vf_explained_var": -5.7312159462696854e-09, "kl": 0.01018442865461111, "entropy": -1.8456313610076904, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10440000, "episodes_total": 3480, "training_iteration": 174, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-39-14", "timestamp": 1570804754, "time_this_iter_s": 230.36347889900208, "time_total_s": 41045.50529742241, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 41045.50529742241, "timesteps_since_restore": 10440000, "iterations_since_restore": 174, "perf": {"cpu_util_percent": 51.52857142857143, "ram_util_percent": 51.50516717325228}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3054.637216163588, "episode_reward_min": 1617.8097711668156, "episode_reward_mean": 2309.134063394056, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.325675384201543, "mean_processing_ms": 1.476061597471345, "mean_inference_ms": 0.8375878027438316}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10483200, "num_steps_sampled": 10500000, "sample_time_ms": 223556.375, "load_time_ms": 10.336, "grad_time_ms": 5070.112, "update_time_ms": 4.665, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 384.08416748046875, "policy_loss": 0.0017204862087965012, "vf_loss": 384.0824279785156, "vf_explained_var": -8.405783624709784e-09, "kl": 0.00944794062525034, "entropy": -1.7994898557662964, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10500000, "episodes_total": 3500, "training_iteration": 175, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-43-00", "timestamp": 1570804980, "time_this_iter_s": 226.58526182174683, "time_total_s": 41272.090559244156, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 41272.090559244156, "timesteps_since_restore": 10500000, "iterations_since_restore": 175, "perf": {"cpu_util_percent": 51.372445820433434, "ram_util_percent": 51.51021671826625}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3054.637216163588, "episode_reward_min": 1617.8097711668156, "episode_reward_mean": 2325.1215299217442, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.3247633479586, "mean_processing_ms": 1.4758386395361094, "mean_inference_ms": 0.8373961014041721}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10543104, "num_steps_sampled": 10560000, "sample_time_ms": 223663.784, "load_time_ms": 10.295, "grad_time_ms": 5070.802, "update_time_ms": 4.616, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 413.54815673828125, "policy_loss": 0.0012772162444889545, "vf_loss": 413.546875, "vf_explained_var": -4.839693534819389e-09, "kl": 0.009891820140182972, "entropy": -1.8770949840545654, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10560000, "episodes_total": 3520, "training_iteration": 176, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-46-50", "timestamp": 1570805210, "time_this_iter_s": 229.45271515846252, "time_total_s": 41501.54327440262, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 41501.54327440262, "timesteps_since_restore": 10560000, "iterations_since_restore": 176, "perf": {"cpu_util_percent": 51.712804878048786, "ram_util_percent": 51.591463414634156}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3054.637216163588, "episode_reward_min": 1617.8097711668156, "episode_reward_mean": 2351.552939156018, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.323892085106135, "mean_processing_ms": 1.4756077989050795, "mean_inference_ms": 0.8372065220748831}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10603008, "num_steps_sampled": 10620000, "sample_time_ms": 223506.598, "load_time_ms": 10.243, "grad_time_ms": 5076.401, "update_time_ms": 4.617, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 391.6576232910156, "policy_loss": 0.0014252109685912728, "vf_loss": 391.6562194824219, "vf_explained_var": -5.094414223805188e-09, "kl": 0.009858991019427776, "entropy": -1.8501771688461304, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10620000, "episodes_total": 3540, "training_iteration": 177, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-50-38", "timestamp": 1570805438, "time_this_iter_s": 228.038245677948, "time_total_s": 41729.58152008057, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 41729.58152008057, "timesteps_since_restore": 10620000, "iterations_since_restore": 177, "perf": {"cpu_util_percent": 51.31846153846154, "ram_util_percent": 51.599076923076936}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3037.2034171631726, "episode_reward_min": 1379.466660685787, "episode_reward_mean": 2387.840205469589, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.323056324836562, "mean_processing_ms": 1.4753834938093247, "mean_inference_ms": 0.8370115137179864}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10662912, "num_steps_sampled": 10680000, "sample_time_ms": 223262.66, "load_time_ms": 10.254, "grad_time_ms": 5078.192, "update_time_ms": 4.713, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 418.8285827636719, "policy_loss": 0.0018733737524598837, "vf_loss": 418.8267517089844, "vf_explained_var": -2.9292881453812925e-09, "kl": 0.010292772203683853, "entropy": -1.896974802017212, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10680000, "episodes_total": 3560, "training_iteration": 178, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-54-27", "timestamp": 1570805667, "time_this_iter_s": 229.4220325946808, "time_total_s": 41959.00355267525, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 41959.00355267525, "timesteps_since_restore": 10680000, "iterations_since_restore": 178, "perf": {"cpu_util_percent": 51.26085626911315, "ram_util_percent": 51.62629969418961}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3027.3897033104518, "episode_reward_min": 1379.466660685787, "episode_reward_mean": 2351.7860060329613, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.32213661438874, "mean_processing_ms": 1.4751260472793246, "mean_inference_ms": 0.8368124718118618}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10722816, "num_steps_sampled": 10740000, "sample_time_ms": 223182.238, "load_time_ms": 10.078, "grad_time_ms": 5076.401, "update_time_ms": 4.718, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 353.7749328613281, "policy_loss": 0.0018383120186626911, "vf_loss": 353.77313232421875, "vf_explained_var": -1.2099233615003868e-08, "kl": 0.007688155397772789, "entropy": -1.8460592031478882, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10740000, "episodes_total": 3580, "training_iteration": 179, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_07-58-14", "timestamp": 1570805894, "time_this_iter_s": 227.16819834709167, "time_total_s": 42186.17175102234, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 42186.17175102234, "timesteps_since_restore": 10740000, "iterations_since_restore": 179, "perf": {"cpu_util_percent": 51.42407407407408, "ram_util_percent": 51.69475308641975}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3027.3897033104518, "episode_reward_min": 1379.466660685787, "episode_reward_mean": 2333.6047019246557, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.321248849609199, "mean_processing_ms": 1.4748735382687588, "mean_inference_ms": 0.8366240309396852}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10782720, "num_steps_sampled": 10800000, "sample_time_ms": 223234.743, "load_time_ms": 10.053, "grad_time_ms": 5077.939, "update_time_ms": 4.6, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 350.7579650878906, "policy_loss": 0.0006976635777391493, "vf_loss": 350.75726318359375, "vf_explained_var": -4.967053879312289e-09, "kl": 0.010407296940684319, "entropy": -1.8338757753372192, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10800000, "episodes_total": 3600, "training_iteration": 180, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-02-02", "timestamp": 1570806122, "time_this_iter_s": 227.72692155838013, "time_total_s": 42413.89867258072, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 42413.89867258072, "timesteps_since_restore": 10800000, "iterations_since_restore": 180, "perf": {"cpu_util_percent": 51.39507692307693, "ram_util_percent": 51.699384615384616}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3027.3897033104518, "episode_reward_min": 1379.466660685787, "episode_reward_mean": 2309.0435616183545, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.32027988142525, "mean_processing_ms": 1.4745946199647024, "mean_inference_ms": 0.8364303488692733}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10842624, "num_steps_sampled": 10860000, "sample_time_ms": 223125.729, "load_time_ms": 10.101, "grad_time_ms": 5081.137, "update_time_ms": 4.595, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 373.5103759765625, "policy_loss": 0.0015793710481375456, "vf_loss": 373.5087890625, "vf_explained_var": -4.58497284583359e-09, "kl": 0.009842636995017529, "entropy": -1.8823915719985962, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10860000, "episodes_total": 3620, "training_iteration": 181, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-05-49", "timestamp": 1570806349, "time_this_iter_s": 227.1251826286316, "time_total_s": 42641.02385520935, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 42641.02385520935, "timesteps_since_restore": 10860000, "iterations_since_restore": 181, "perf": {"cpu_util_percent": 51.2641975308642, "ram_util_percent": 51.75833333333333}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3027.3897033104518, "episode_reward_min": 1379.466660685787, "episode_reward_mean": 2333.5278943553653, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.319407906525791, "mean_processing_ms": 1.4743501908323235, "mean_inference_ms": 0.836247519749561}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10902528, "num_steps_sampled": 10920000, "sample_time_ms": 223405.266, "load_time_ms": 10.035, "grad_time_ms": 5079.476, "update_time_ms": 4.6, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 425.79461669921875, "policy_loss": 0.001901145908050239, "vf_loss": 425.792724609375, "vf_explained_var": -3.8208107788761936e-09, "kl": 0.010114246979355812, "entropy": -1.959778904914856, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10920000, "episodes_total": 3640, "training_iteration": 182, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-09-40", "timestamp": 1570806580, "time_this_iter_s": 230.62169885635376, "time_total_s": 42871.645554065704, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 42871.645554065704, "timesteps_since_restore": 10920000, "iterations_since_restore": 182, "perf": {"cpu_util_percent": 51.54924012158056, "ram_util_percent": 51.79574468085107}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3016.9403554476085, "episode_reward_min": 1632.9454111848383, "episode_reward_mean": 2312.9844463865734, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.31852330585841, "mean_processing_ms": 1.4741123660214692, "mean_inference_ms": 0.8360715576549578}, "off_policy_estimator": {}, "info": {"num_steps_trained": 10962432, "num_steps_sampled": 10980000, "sample_time_ms": 223419.137, "load_time_ms": 10.077, "grad_time_ms": 5083.09, "update_time_ms": 4.654, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 385.3238220214844, "policy_loss": 0.0021609573159366846, "vf_loss": 385.3216552734375, "vf_explained_var": -6.750098702212881e-09, "kl": 0.012110111303627491, "entropy": -1.9301774501800537, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 10980000, "episodes_total": 3660, "training_iteration": 183, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-13-29", "timestamp": 1570806809, "time_this_iter_s": 228.80314350128174, "time_total_s": 43100.448697566986, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 43100.448697566986, "timesteps_since_restore": 10980000, "iterations_since_restore": 183, "perf": {"cpu_util_percent": 51.340797546012276, "ram_util_percent": 51.79877300613497}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3081.135910454933, "episode_reward_min": 1632.9454111848383, "episode_reward_mean": 2356.752828140425, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.317806953497146, "mean_processing_ms": 1.4739050282046844, "mean_inference_ms": 0.835913804072241}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11022336, "num_steps_sampled": 11040000, "sample_time_ms": 223653.408, "load_time_ms": 10.053, "grad_time_ms": 5082.151, "update_time_ms": 4.676, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 433.0020751953125, "policy_loss": 0.0017846066039055586, "vf_loss": 433.00030517578125, "vf_explained_var": -4.202891812354892e-09, "kl": 0.009497447870671749, "entropy": -1.9684594869613647, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11040000, "episodes_total": 3680, "training_iteration": 184, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-17-21", "timestamp": 1570807041, "time_this_iter_s": 232.6968982219696, "time_total_s": 43333.145595788956, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 43333.145595788956, "timesteps_since_restore": 11040000, "iterations_since_restore": 184, "perf": {"cpu_util_percent": 53.984939759036145, "ram_util_percent": 51.96596385542169}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3081.135910454933, "episode_reward_min": 1667.394458938387, "episode_reward_mean": 2381.375885498674, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.317152155339651, "mean_processing_ms": 1.4737214327130448, "mean_inference_ms": 0.835760440683614}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11082240, "num_steps_sampled": 11100000, "sample_time_ms": 223959.642, "load_time_ms": 10.005, "grad_time_ms": 5081.88, "update_time_ms": 4.57, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 392.8614807128906, "policy_loss": 0.0022623285185545683, "vf_loss": 392.8591613769531, "vf_explained_var": -4.202891812354892e-09, "kl": 0.007646601647138596, "entropy": -1.9432039260864258, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11100000, "episodes_total": 3700, "training_iteration": 185, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-21-11", "timestamp": 1570807271, "time_this_iter_s": 229.64303302764893, "time_total_s": 43562.788628816605, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 43562.788628816605, "timesteps_since_restore": 11100000, "iterations_since_restore": 185, "perf": {"cpu_util_percent": 51.84054878048781, "ram_util_percent": 55.32225609756098}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3098.8350768758964, "episode_reward_min": 1625.2171266307498, "episode_reward_mean": 2379.2991182695537, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.31657170797969, "mean_processing_ms": 1.4735733613793303, "mean_inference_ms": 0.8356162438051601}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11142144, "num_steps_sampled": 11160000, "sample_time_ms": 223915.74, "load_time_ms": 10.021, "grad_time_ms": 5080.988, "update_time_ms": 4.505, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 383.97369384765625, "policy_loss": 0.0026963071431964636, "vf_loss": 383.97100830078125, "vf_explained_var": -5.4764952572838865e-09, "kl": 0.009138607420027256, "entropy": -1.9157949686050415, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11160000, "episodes_total": 3720, "training_iteration": 186, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-25-00", "timestamp": 1570807500, "time_this_iter_s": 229.00459384918213, "time_total_s": 43791.79322266579, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 43791.79322266579, "timesteps_since_restore": 11160000, "iterations_since_restore": 186, "perf": {"cpu_util_percent": 51.5743119266055, "ram_util_percent": 55.998165137614684}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3098.8350768758964, "episode_reward_min": 1625.2171266307498, "episode_reward_mean": 2351.530405847605, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.31592791592413, "mean_processing_ms": 1.4734123641842751, "mean_inference_ms": 0.8354681170802638}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11202048, "num_steps_sampled": 11220000, "sample_time_ms": 224050.321, "load_time_ms": 10.052, "grad_time_ms": 5078.057, "update_time_ms": 4.427, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 384.4533996582031, "policy_loss": 0.0019912065472453833, "vf_loss": 384.4513854980469, "vf_explained_var": -1.273603555951297e-09, "kl": 0.011389504186809063, "entropy": -1.958509922027588, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11220000, "episodes_total": 3740, "training_iteration": 187, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-28-50", "timestamp": 1570807730, "time_this_iter_s": 229.3536684513092, "time_total_s": 44021.146891117096, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 44021.146891117096, "timesteps_since_restore": 11220000, "iterations_since_restore": 187, "perf": {"cpu_util_percent": 51.22660550458715, "ram_util_percent": 56.01070336391438}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3098.8350768758964, "episode_reward_min": 1625.2171266307498, "episode_reward_mean": 2381.9223338432475, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.315320223039513, "mean_processing_ms": 1.4732345124600834, "mean_inference_ms": 0.8353162588568841}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11261952, "num_steps_sampled": 11280000, "sample_time_ms": 223943.815, "load_time_ms": 10.064, "grad_time_ms": 5074.833, "update_time_ms": 4.365, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 435.67926025390625, "policy_loss": 0.0012859683483839035, "vf_loss": 435.6779479980469, "vf_explained_var": -5.4764952572838865e-09, "kl": 0.009204023517668247, "entropy": -2.0398941040039062, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11280000, "episodes_total": 3760, "training_iteration": 188, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-32-38", "timestamp": 1570807958, "time_this_iter_s": 228.32455682754517, "time_total_s": 44249.47144794464, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 44249.47144794464, "timesteps_since_restore": 11280000, "iterations_since_restore": 188, "perf": {"cpu_util_percent": 51.424233128834366, "ram_util_percent": 56.09263803680983}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3098.8350768758964, "episode_reward_min": 1625.2171266307498, "episode_reward_mean": 2345.661368938575, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.314600932128668, "mean_processing_ms": 1.473040611884727, "mean_inference_ms": 0.8351520134882991}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11321856, "num_steps_sampled": 11340000, "sample_time_ms": 224075.614, "load_time_ms": 10.103, "grad_time_ms": 5077.491, "update_time_ms": 4.275, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 373.9120788574219, "policy_loss": 0.0016731752548366785, "vf_loss": 373.910400390625, "vf_explained_var": 3.82081061234274e-10, "kl": 0.012111539952456951, "entropy": -1.958390235900879, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11340000, "episodes_total": 3780, "training_iteration": 189, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-36-26", "timestamp": 1570808186, "time_this_iter_s": 228.5127408504486, "time_total_s": 44477.98418879509, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 44477.98418879509, "timesteps_since_restore": 11340000, "iterations_since_restore": 189, "perf": {"cpu_util_percent": 51.38865030674847, "ram_util_percent": 56.099693251533765}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3098.8350768758964, "episode_reward_min": 1625.2171266307498, "episode_reward_mean": 2342.3243566614283, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.313873394351817, "mean_processing_ms": 1.4728552651667477, "mean_inference_ms": 0.8349895846908116}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11381760, "num_steps_sampled": 11400000, "sample_time_ms": 224234.021, "load_time_ms": 10.158, "grad_time_ms": 5076.36, "update_time_ms": 4.321, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 388.7669982910156, "policy_loss": 0.004370197653770447, "vf_loss": 388.7625732421875, "vf_explained_var": -1.0061468103117477e-08, "kl": 0.014821565710008144, "entropy": -1.9951106309890747, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11400000, "episodes_total": 3800, "training_iteration": 190, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-40-16", "timestamp": 1570808416, "time_this_iter_s": 229.3007504940033, "time_total_s": 44707.28493928909, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 44707.28493928909, "timesteps_since_restore": 11400000, "iterations_since_restore": 190, "perf": {"cpu_util_percent": 51.472477064220186, "ram_util_percent": 56.148929663608556}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3054.7912484319218, "episode_reward_min": 1644.044391956315, "episode_reward_mean": 2351.491244229064, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.313113009251372, "mean_processing_ms": 1.4726607383911072, "mean_inference_ms": 0.8348284049296928}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11441664, "num_steps_sampled": 11460000, "sample_time_ms": 224428.723, "load_time_ms": 10.14, "grad_time_ms": 5074.62, "update_time_ms": 4.229, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 394.8280029296875, "policy_loss": 0.002570396289229393, "vf_loss": 394.825439453125, "vf_explained_var": -1.1971873270510969e-08, "kl": 0.012304340489208698, "entropy": -2.016383409500122, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11460000, "episodes_total": 3820, "training_iteration": 191, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-44-05", "timestamp": 1570808645, "time_this_iter_s": 229.0536184310913, "time_total_s": 44936.338557720184, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 44936.338557720184, "timesteps_since_restore": 11460000, "iterations_since_restore": 191, "perf": {"cpu_util_percent": 51.11042944785276, "ram_util_percent": 56.1638036809816}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3097.0499886949797, "episode_reward_min": 1644.044391956315, "episode_reward_mean": 2380.7444319412434, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.312370471447091, "mean_processing_ms": 1.4724729868923594, "mean_inference_ms": 0.8346652715441004}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11501568, "num_steps_sampled": 11520000, "sample_time_ms": 224340.091, "load_time_ms": 10.19, "grad_time_ms": 5078.363, "update_time_ms": 4.206, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 433.66241455078125, "policy_loss": 0.002431400353088975, "vf_loss": 433.6600036621094, "vf_explained_var": -4.457612501340691e-09, "kl": 0.012421557679772377, "entropy": -2.087833881378174, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11520000, "episodes_total": 3840, "training_iteration": 192, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-47-55", "timestamp": 1570808875, "time_this_iter_s": 229.77351784706116, "time_total_s": 45166.112075567245, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 45166.112075567245, "timesteps_since_restore": 11520000, "iterations_since_restore": 192, "perf": {"cpu_util_percent": 51.34634146341464, "ram_util_percent": 56.01493902439025}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3097.0499886949797, "episode_reward_min": 1644.044391956315, "episode_reward_mean": 2368.369046283986, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.311642060199249, "mean_processing_ms": 1.4722870027966446, "mean_inference_ms": 0.8345068242874324}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11561472, "num_steps_sampled": 11580000, "sample_time_ms": 224396.483, "load_time_ms": 10.2, "grad_time_ms": 5079.43, "update_time_ms": 4.198, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 420.4528503417969, "policy_loss": 0.0016009387327358127, "vf_loss": 420.45123291015625, "vf_explained_var": -6.240657324241283e-09, "kl": 0.01214947272092104, "entropy": -2.0898144245147705, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11580000, "episodes_total": 3860, "training_iteration": 193, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-51-44", "timestamp": 1570809104, "time_this_iter_s": 229.3771834373474, "time_total_s": 45395.48925900459, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 45395.48925900459, "timesteps_since_restore": 11580000, "iterations_since_restore": 193, "perf": {"cpu_util_percent": 51.203058103975536, "ram_util_percent": 56.08685015290523}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3097.0499886949797, "episode_reward_min": 1537.9424905465285, "episode_reward_mean": 2424.6104230103965, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.311017146456566, "mean_processing_ms": 1.4721219164898418, "mean_inference_ms": 0.8343568640810725}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11621376, "num_steps_sampled": 11640000, "sample_time_ms": 224276.656, "load_time_ms": 10.257, "grad_time_ms": 5081.77, "update_time_ms": 4.139, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 466.02496337890625, "policy_loss": 0.0033219954930245876, "vf_loss": 466.0216369628906, "vf_explained_var": -4.58497284583359e-09, "kl": 0.016518140211701393, "entropy": -2.150561809539795, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11640000, "episodes_total": 3880, "training_iteration": 194, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-55-35", "timestamp": 1570809335, "time_this_iter_s": 231.52236151695251, "time_total_s": 45627.011620521545, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 45627.011620521545, "timesteps_since_restore": 11640000, "iterations_since_restore": 194, "perf": {"cpu_util_percent": 51.57885196374622, "ram_util_percent": 56.09848942598189}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3097.0499886949797, "episode_reward_min": 1537.9424905465285, "episode_reward_mean": 2421.0279043025266, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.310349665648566, "mean_processing_ms": 1.4719305297110157, "mean_inference_ms": 0.8341999021198612}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11681280, "num_steps_sampled": 11700000, "sample_time_ms": 224016.541, "load_time_ms": 10.279, "grad_time_ms": 5085.547, "update_time_ms": 4.246, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 389.2752685546875, "policy_loss": 0.0028628669679164886, "vf_loss": 389.2723693847656, "vf_explained_var": -8.915225002681382e-09, "kl": 0.012709049507975578, "entropy": -2.0523343086242676, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11700000, "episodes_total": 3900, "training_iteration": 195, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_08-59-23", "timestamp": 1570809563, "time_this_iter_s": 227.08076214790344, "time_total_s": 45854.09238266945, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 45854.09238266945, "timesteps_since_restore": 11700000, "iterations_since_restore": 195, "perf": {"cpu_util_percent": 51.35679012345678, "ram_util_percent": 55.60061728395061}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3097.0499886949797, "episode_reward_min": 1537.9424905465285, "episode_reward_mean": 2409.73189234466, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.309671688070273, "mean_processing_ms": 1.4717520710850662, "mean_inference_ms": 0.8340386012548988}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11741184, "num_steps_sampled": 11760000, "sample_time_ms": 223978.794, "load_time_ms": 10.278, "grad_time_ms": 5087.212, "update_time_ms": 4.308, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 383.9013671875, "policy_loss": 0.0027107472997158766, "vf_loss": 383.8986511230469, "vf_explained_var": 0.0, "kl": 0.012237018905580044, "entropy": -2.0226101875305176, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11760000, "episodes_total": 3920, "training_iteration": 196, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_09-03-11", "timestamp": 1570809791, "time_this_iter_s": 228.6441512107849, "time_total_s": 46082.736533880234, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 46082.736533880234, "timesteps_since_restore": 11760000, "iterations_since_restore": 196, "perf": {"cpu_util_percent": 51.12147239263804, "ram_util_percent": 54.19907975460123}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3067.8270147093335, "episode_reward_min": 1537.9424905465285, "episode_reward_mean": 2373.2781452362055, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.309009505881488, "mean_processing_ms": 1.4715678069903064, "mean_inference_ms": 0.8338863915694975}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11801088, "num_steps_sampled": 11820000, "sample_time_ms": 223971.878, "load_time_ms": 10.263, "grad_time_ms": 5086.107, "update_time_ms": 4.304, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 381.7317810058594, "policy_loss": 0.001686975359916687, "vf_loss": 381.7301025390625, "vf_explained_var": -1.7830449339228949e-09, "kl": 0.008802078664302826, "entropy": -2.056385040283203, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11820000, "episodes_total": 3940, "training_iteration": 197, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_09-07-00", "timestamp": 1570810020, "time_this_iter_s": 229.27349042892456, "time_total_s": 46312.01002430916, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 46312.01002430916, "timesteps_since_restore": 11820000, "iterations_since_restore": 197, "perf": {"cpu_util_percent": 51.42752293577981, "ram_util_percent": 54.199999999999996}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3067.8270147093335, "episode_reward_min": 1537.9424905465285, "episode_reward_mean": 2362.2314232442523, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.308319224551878, "mean_processing_ms": 1.47140686606288, "mean_inference_ms": 0.8337354392680725}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11860992, "num_steps_sampled": 11880000, "sample_time_ms": 224042.792, "load_time_ms": 10.249, "grad_time_ms": 5088.39, "update_time_ms": 4.227, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 409.4963073730469, "policy_loss": 0.0022892404813319445, "vf_loss": 409.4940185546875, "vf_explained_var": -6.8774590467057806e-09, "kl": 0.012833282351493835, "entropy": -2.0907657146453857, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11880000, "episodes_total": 3960, "training_iteration": 198, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_09-10-50", "timestamp": 1570810250, "time_this_iter_s": 229.05612421035767, "time_total_s": 46541.066148519516, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 46541.066148519516, "timesteps_since_restore": 11880000, "iterations_since_restore": 198, "perf": {"cpu_util_percent": 51.28348623853211, "ram_util_percent": 54.2217125382263}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3067.8270147093335, "episode_reward_min": 1617.0572138047326, "episode_reward_mean": 2309.6644856533026, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.307533921500677, "mean_processing_ms": 1.4712477307166443, "mean_inference_ms": 0.833577081298144}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11920896, "num_steps_sampled": 11940000, "sample_time_ms": 224030.812, "load_time_ms": 10.206, "grad_time_ms": 5207.114, "update_time_ms": 4.161, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 391.09295654296875, "policy_loss": 0.0038364955689758062, "vf_loss": 391.0890808105469, "vf_explained_var": -7.641621557752387e-09, "kl": 0.013565467670559883, "entropy": -2.065732955932617, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": false, "timesteps_total": 11940000, "episodes_total": 3980, "training_iteration": 199, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_09-14-39", "timestamp": 1570810479, "time_this_iter_s": 229.57902789115906, "time_total_s": 46770.645176410675, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 46770.645176410675, "timesteps_since_restore": 11940000, "iterations_since_restore": 199, "perf": {"cpu_util_percent": 52.2617737003058, "ram_util_percent": 54.309174311926604}, "num_healthy_workers": 2, "trial_id": "4dce3444"} +{"episode_reward_max": 3067.8270147093335, "episode_reward_min": 1384.8900407828025, "episode_reward_mean": 2324.433615775931, "episode_len_mean": 3000.0, "episodes_this_iter": 20, "policy_reward_mean": {}, "custom_metrics": {}, "sampler_perf": {"mean_env_wait_ms": 5.306923250069895, "mean_processing_ms": 1.4711140704663632, "mean_inference_ms": 0.8334424370031818}, "off_policy_estimator": {}, "info": {"num_steps_trained": 11980800, "num_steps_sampled": 12000000, "sample_time_ms": 224493.937, "load_time_ms": 10.2, "grad_time_ms": 5206.619, "update_time_ms": 4.328, "learner": {"default_policy": {"cur_kl_coeff": 0.0, "cur_lr": 4.999999873689376e-05, "total_loss": 414.9051818847656, "policy_loss": 0.0038591448683291674, "vf_loss": 414.9013366699219, "vf_explained_var": -6.240657324241283e-09, "kl": 0.01615159772336483, "entropy": -2.1259605884552, "entropy_coeff": 0.0}}}, "timesteps_this_iter": 60000, "done": true, "timesteps_total": 12000000, "episodes_total": 4000, "training_iteration": 200, "experiment_id": "fec93e2e40d94966abd856ec126a6b85", "date": "2019-10-11_09-18-33", "timestamp": 1570810713, "time_this_iter_s": 233.92884373664856, "time_total_s": 47004.57402014732, "pid": 21927, "hostname": "ellie", "node_ip": "10.0.0.158", "config": {"monitor": false, "log_level": "INFO", "callbacks": {"on_episode_start": null, "on_episode_step": null, "on_episode_end": null, "on_sample_end": null, "on_train_result": null, "on_postprocess_traj": null}, "ignore_worker_failures": false, "log_sys_usage": true, "model": {"conv_filters": null, "conv_activation": "relu", "fcnet_activation": "tanh", "fcnet_hiddens": [3, 3], "free_log_std": false, "no_final_linear": false, "vf_share_layers": true, "use_lstm": false, "max_seq_len": 20, "lstm_cell_size": 256, "lstm_use_prev_action_reward": false, "state_shape": null, "framestack": true, "dim": 84, "grayscale": false, "zero_mean": true, "custom_preprocessor": null, "custom_model": null, "custom_options": {}}, "optimizer": {}, "gamma": 0.999, "horizon": 3000, "soft_horizon": false, "env_config": {"flow_params": "{\n \"env\": {\n \"additional_params\": {\n \"max_accel\": 1,\n \"max_decel\": 1,\n \"ring_length\": [\n 220,\n 270\n ]\n },\n \"clip_actions\": false,\n \"evaluate\": false,\n \"horizon\": 3000,\n \"sims_per_step\": 1,\n \"warmup_steps\": 750\n },\n \"env_name\": \"flow.envs.ring.wave_attenuation.WaveAttenuationPOEnv\",\n \"exp_tag\": \"stabilizing_the_ring\",\n \"initial\": {\n \"additional_params\": {},\n \"bunching\": 0,\n \"edges_distribution\": \"all\",\n \"lanes_distribution\": Infinity,\n \"min_gap\": 0,\n \"perturbation\": 0.0,\n \"shuffle\": false,\n \"spacing\": \"uniform\",\n \"x0\": 0\n },\n \"net\": {\n \"additional_params\": {\n \"lanes\": 1,\n \"length\": 260,\n \"resolution\": 40,\n \"speed_limit\": 30\n },\n \"inflows\": {\n \"_InFlows__flows\": []\n },\n \"osm_path\": null,\n \"template\": null\n },\n \"network\": \"flow.networks.ring.RingNetwork\",\n \"sim\": {\n \"color_vehicles\": true,\n \"emission_path\": null,\n \"lateral_resolution\": null,\n \"no_step_log\": true,\n \"num_clients\": 1,\n \"overtake_right\": false,\n \"port\": null,\n \"print_warnings\": true,\n \"pxpm\": 2,\n \"render\": false,\n \"restart_instance\": false,\n \"save_render\": false,\n \"seed\": null,\n \"show_radius\": false,\n \"sight_radius\": 25,\n \"sim_step\": 0.1,\n \"teleport_time\": -1\n },\n \"simulator\": \"traci\",\n \"veh\": [\n {\n \"acceleration_controller\": [\n \"IDMController\",\n {\n \"noise\": 0.2\n }\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 0,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 21,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"human\"\n },\n {\n \"acceleration_controller\": [\n \"RLController\",\n {}\n ],\n \"car_following_params\": {\n \"controller_params\": {\n \"accel\": 2.6,\n \"carFollowModel\": \"IDM\",\n \"decel\": 4.5,\n \"impatience\": 0.5,\n \"maxSpeed\": 30,\n \"minGap\": 2.5,\n \"sigma\": 0.5,\n \"speedDev\": 0.1,\n \"speedFactor\": 1.0,\n \"tau\": 1.0\n },\n \"speed_mode\": 25\n },\n \"initial_speed\": 0,\n \"lane_change_controller\": [\n \"SimLaneChangeController\",\n {}\n ],\n \"lane_change_params\": {\n \"controller_params\": {\n \"laneChangeModel\": \"LC2013\",\n \"lcCooperative\": \"1.0\",\n \"lcKeepRight\": \"1.0\",\n \"lcSpeedGain\": \"1.0\",\n \"lcStrategic\": \"1.0\"\n },\n \"lane_change_mode\": 512\n },\n \"num_vehicles\": 1,\n \"routing_controller\": [\n \"ContinuousRouter\",\n {}\n ],\n \"veh_id\": \"rl\"\n }\n ]\n}", "run": "PPO"}, "env": "WaveAttenuationPOEnv-v0", "clip_rewards": null, "clip_actions": false, "preprocessor_pref": "deepmind", "lr": 5e-05, "evaluation_interval": null, "evaluation_num_episodes": 10, "evaluation_config": {}, "num_workers": 2, "num_gpus": 0, "num_cpus_per_worker": 1, "num_gpus_per_worker": 0, "custom_resources_per_worker": {}, "num_cpus_for_driver": 1, "num_envs_per_worker": 1, "sample_batch_size": 200, "train_batch_size": 60000, "batch_mode": "truncate_episodes", "sample_async": false, "observation_filter": "NoFilter", "synchronize_filters": true, "tf_session_args": {"intra_op_parallelism_threads": 2, "inter_op_parallelism_threads": 2, "gpu_options": {"allow_growth": true}, "log_device_placement": false, "device_count": {"CPU": 1}, "allow_soft_placement": true}, "local_tf_session_args": {"intra_op_parallelism_threads": 8, "inter_op_parallelism_threads": 8}, "compress_observations": false, "collect_metrics_timeout": 180, "metrics_smoothing_episodes": 100, "remote_worker_envs": false, "remote_env_batch_wait_ms": 0, "min_iter_time_s": 0, "timesteps_per_iteration": 0, "seed": null, "input": "sampler", "input_evaluation": ["is", "wis"], "postprocess_inputs": false, "shuffle_buffer_size": 0, "output": null, "output_compress_columns": ["obs", "new_obs"], "output_max_file_size": 67108864, "multiagent": {"policies": {}, "policy_mapping_fn": null, "policies_to_train": null}, "use_gae": true, "lambda": 0.97, "kl_coeff": 0.2, "sgd_minibatch_size": 128, "shuffle_sequences": true, "num_sgd_iter": 10, "lr_schedule": null, "vf_share_layers": false, "vf_loss_coeff": 1.0, "entropy_coeff": 0.0, "entropy_coeff_schedule": null, "clip_param": 0.3, "vf_clip_param": 10.0, "grad_clip": null, "kl_target": 0.02, "simple_optimizer": false}, "time_since_restore": 47004.57402014732, "timesteps_since_restore": 12000000, "iterations_since_restore": 200, "perf": {"cpu_util_percent": 53.853592814371254, "ram_util_percent": 54.06766467065868}, "num_healthy_workers": 2, "trial_id": "4dce3444"} diff --git a/tutorials/img/ring_scenario.png b/tutorials/img/ring_network.png similarity index 100% rename from tutorials/img/ring_scenario.png rename to tutorials/img/ring_network.png diff --git a/tutorials/tutorial00_flow.ipynb b/tutorials/tutorial00_flow.ipynb index de3682c3e..689a1dfdd 100644 --- a/tutorials/tutorial00_flow.ipynb +++ b/tutorials/tutorial00_flow.ipynb @@ -65,10 +65,11 @@ "flow\n", "├── docs # some random documents, don't worry about it\n", "├── examples # a lot of example codes using Flow -- this is where you want to head once you're done with the tutorials and want to start doing some real code\n", - "│ ├── aimsun # examples using just Aimsun, without training \n", - "│ ├── rllab # examples of training with rllab (avoid rllab, we are going to deprecate it soon!)\n", - "│ ├── rllib # examples of training with RLlib\n", - "│ └── sumo # examples using just SUMO, without training. You can run these right away to see what the networks look like!\n", + "│ └── exp_configs # configuration of the all the examples \n", + "│ ├── non_rl # configurations of examples with simulations (e.g. either SUMO or Aimsun) without any Reinforcement Learning\n", + "│ └── rl\n", + "│ ├── singleagent # configurations of examples with training single agent RL contollers\n", + "│ └── multiagent # configurations of examples with training multi agent RL contollers\n", "├── flow\n", "│ ├── benchmarks # several custom networks and configurations on which you can evaluate and compare different RL algorithms\n", "│ ├── controllers # implementations of controllers for the vehicles (IDM, Follower-Stopper...)\n", @@ -79,7 +80,7 @@ "│ │ ├── traffic_light # logic for the traffic lights\n", "│ │ └── vehicle # logic for the vehicles\n", "│ ├── envs # environments (where states, actions and rewards are handled)\n", - "│ ├── multiagent_envs # multi-agent environments\n", + "│ │ └── multiagent # multi-agent environments\n", "│ ├── renderer # pyglet renderer\n", "│ ├── networks # networks (ie road networks)\n", "│ ├── utils # the files that don't fit anywhere else\n", @@ -109,7 +110,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.0" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/tutorials/tutorial01_sumo.ipynb b/tutorials/tutorial01_sumo.ipynb index f7ae11cd5..18e7cdf26 100644 --- a/tutorials/tutorial01_sumo.ipynb +++ b/tutorials/tutorial01_sumo.ipynb @@ -6,15 +6,15 @@ "source": [ "# Tutorial 01: Running Sumo Simulations\n", "\n", - "This tutorial walks through the process of running non-RL traffic simulations in Flow. Simulations of this form act as non-autonomous baselines and depict the behavior of human dynamics on a network. Similar simulations may also be used to evaluate the performance of hand-designed controllers on a network. This tutorial focuses primarily on the former use case, while an example of the latter may be found in `exercise07_controllers.ipynb`.\n", + "This tutorial walks through the process of running non-RL traffic simulations in Flow. Simulations of this form act as non-autonomous baselines and depict the behavior of human dynamics on a network. Similar simulations may also be used to evaluate the performance of hand-designed controllers on a network. This tutorial focuses primarily on the former use case, while an example of the latter may be found in `tutorial09_controllers.ipynb`.\n", "\n", - "In this exercise, we simulate a initially perturbed single lane ring road. We witness in simulation that as time advances the initially perturbations do not dissipate, but instead propagates and expands until vehicles are forced to periodically stop and accelerate. For more information on this behavior, we refer the reader to the following article [1].\n", + "In this tutorial, we simulate a initially perturbed single lane ring road. We witness in simulation that as time advances the initially perturbations do not dissipate, but instead propagates and expands until vehicles are forced to periodically stop and accelerate. For more information on this behavior, we refer the reader to the following article [1].\n", "\n", "## 1. Components of a Simulation\n", "All simulations, both in the presence and absence of RL, require two components: a *network*, and an *environment*. Networks describe the features of the transportation network used in simulation. This includes the positions and properties of nodes and edges constituting the lanes and junctions, as well as properties of the vehicles, traffic lights, inflows, etc. in the network. Environments, on the other hand, initialize, reset, and advance simulations, and act the primary interface between the reinforcement learning algorithm and the network. Moreover, custom environments may be used to modify the dynamical features of an network.\n", "\n", "## 2. Setting up a Network\n", - "Flow contains a plethora of pre-designed networks used to replicate highways, intersections, and merges in both closed and open settings. All these networks are located in flow/networks. In order to recreate a ring road network, we begin by importing the network `RingNetwork`." + "Flow contains a plethora of pre-designed networks used to replicate highways, intersections, and merges in both closed and open settings. All these networks are located in `flow/networks`. In order to recreate a ring road network, we begin by importing the network `RingNetwork`." ] }, { @@ -150,7 +150,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Importing the `ADDITIONAL_NET_PARAMS` dict from the ring road network, we see that the required parameters are:\n", + "Importing the `ADDITIONAL_NET_PARAMS` dictionary from the ring road network, we see that the required parameters are:\n", "\n", "* **length**: length of the ring road\n", "* **lanes**: number of lanes\n", @@ -158,7 +158,7 @@ "* **resolution**: resolution of the curves on the ring. Setting this value to 1 converts the ring to a diamond.\n", "\n", "\n", - "At times, other inputs may be needed from `NetParams` to recreate proper network features/behavior. These requirements can be founded in the network's documentation. For the ring road, no attributes are needed aside from the `additional_params` terms. Furthermore, for this exercise, we use the network's default parameters when creating the `NetParams` object." + "At times, other inputs may be needed from `NetParams` to recreate proper network features/behavior. These requirements can be found in the network's documentation. For the ring road, no attributes are needed aside from the `additional_params` terms. Furthermore, for this tutorial, we use the network's default parameters when creating the `NetParams` object." ] }, { @@ -198,7 +198,7 @@ "source": [ "### 2.5 TrafficLightParams\n", "\n", - "`TrafficLightParams` are used to describe the positions and types of traffic lights in the network. These inputs are outside the scope of this tutorial, and instead are covered in `exercise06_traffic_lights.ipynb`. For our example, we create an empty `TrafficLightParams` object, thereby ensuring that none are placed on any nodes." + "`TrafficLightParams` are used to describe the positions and types of traffic lights in the network. These inputs are outside the scope of this tutorial, and instead are covered in `tutorial10_traffic_lights.ipynb`. For our example, we create an empty `TrafficLightParams` object, thereby ensuring that none are placed on any nodes." ] }, { @@ -218,7 +218,7 @@ "source": [ "## 3. Setting up an Environment\n", "\n", - "Several envionrments in Flow exist to train autonomous agents of different forms (e.g. autonomous vehicles, traffic lights) to perform a variety of different tasks. These environments are often network or task specific; however, some can be deployed on an ambiguous set of networks as well. One such environment, `AccelEnv`, may be used to train a variable number of vehicles in a fully observable network with a *static* number of vehicles." + "Several envionrments in Flow exist to train autonomous agents of different forms (e.g. autonomous vehicles, traffic lights) to perform a variety of different tasks. These environments are often network- or task-specific; however, some can be deployed on an ambiguous set of networks as well. One such environment, `AccelEnv`, may be used to train a variable number of vehicles in a fully observable network with a *static* number of vehicles." ] }, { @@ -234,14 +234,32 @@ "cell_type": "markdown", "metadata": {}, "source": [ + "Although we will not be training any autonomous agents in this tutorial, the use of an environment allows us to view the cumulative reward simulation rollouts receive in the absence of autonomy.\n", + "\n", "Although we will not be training any autonomous agents in this exercise, the use of an environment allows us to view the cumulative reward simulation rollouts receive in the absence of autonomy.\n", "\n", - "Envrionments in Flow are parametrized by three components:\n", - "* `EnvParams`\n", + "Envrionments in Flow are parametrized by several components, including the following attributes:\n", + "* `sim_params`\n", + "* `env_params`\n", + "* `network`\n", + "* `net_params`\n", + "* `initial_config`\n", + "* `network`\n", + "* `simulator`\n", + "\n", + "where `sim_params`, `env_params`, and `network` are the primary parameters of an environment. For the full list of attributes, please check `class Env` in `flow/envs/base.py`.\n", + "\n", + "Sumo envrionments in Flow are parametrized by three components:\n", "* `SumoParams`\n", + "* `EnvParams`\n", "* `Network`\n", "\n", "### 3.1 SumoParams\n", + "`SumoParams` specifies simulation-specific variables (e.g. `SumoParams` and `AimsunParams` are the variables related to SUMO and Aimsun simulator, respectively). These variables maay include the length a simulation step (in seconds), whether to render the GUI when running the experiment, and other variables. For this example, we consider a SUMO simulation, step length of 0.1s, and activate the GUI.\n", + "\n", + "Another useful parameter is `emission_path`, which is used to specify the path where the emissions output will be generated. They contain a lot of information about the simulation, for instance the position and speed of each car at each time step. If you do not specify any emission path, the emission file will not be generated. More on this in Section 5.\n", + "\n", + "### 3.1 SumoParams\n", "`SumoParams` specifies simulation-specific variables. These variables include the length a simulation step (in seconds) and whether to render the GUI when running the experiment. For this example, we consider a simulation step length of 0.1s and activate the GUI.\n", "\n", "Another useful parameter is `emission_path`, which is used to specify the path where the emissions output will be generated. They contain a lot of information about the simulation, for instance the position and speed of each car at each time step. If you do not specify any emission path, the emission file will not be generated. More on this in Section 5." @@ -255,7 +273,7 @@ "source": [ "from flow.core.params import SumoParams\n", "\n", - "sumo_params = SumoParams(sim_step=0.1, render=True, emission_path='data')" + "sim_params = SumoParams(sim_step=0.1, render=True, emission_path='data')" ] }, { @@ -264,7 +282,7 @@ "source": [ "### 3.2 EnvParams\n", "\n", - "`EnvParams` specify environment and experiment-specific parameters that either affect the training process or the dynamics of various components within the network. Much like `NetParams`, the attributes associated with this parameter are mostly environment specific, and can be found in the environment's `ADDITIONAL_ENV_PARAMS` dictionary." + "`EnvParams` specify environment and experiment-specific parameters that either affect the training process or the dynamics of various components within the network. Much like `NetParams`, the attributes associated with this parameter are mostly environment-specific, and can be found in the environment's `ADDITIONAL_ENV_PARAMS` dictionary." ] }, { @@ -317,7 +335,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "These objects may be used to simulate rollouts in the absence of reinforcement learning agents, as well as acquire behaviors and rewards that may be used as a baseline with which to compare the performance of the learning agent. In this case, we choose to run our experiment for one rollout consisting of 3000 steps (300 s).\n", + "This object may be used to simulate rollouts in the absence of reinforcement learning agents, as well as acquire behaviors and rewards that may be used as a baseline with which to compare the performance of the learning agent. In this case, we choose to run our experiment for one rollout consisting of 3000 steps (300 s).\n", "\n", "**Note**: When executing the below code, remeber to click on the Play button after the GUI is rendered." ] @@ -328,21 +346,25 @@ "metadata": {}, "outputs": [], "source": [ - "# create the network object\n", - "network = RingNetwork(name=\"ring_example\",\n", - " vehicles=vehicles,\n", - " net_params=net_params,\n", - " initial_config=initial_config,\n", - " traffic_lights=traffic_lights)\n", - "\n", - "# create the environment object\n", - "env = AccelEnv(env_params, sumo_params, network)\n", + "flow_params = dict(\n", + " exp_tag='ring_example',\n", + " env_name=AccelEnv,\n", + " network=RingNetwork,\n", + " simulator='traci',\n", + " sim=sim_params,\n", + " env=env_params,\n", + " net=net_params,\n", + " veh=vehicles,\n", + " initial=initial_config,\n", + " tls=traffic_lights,\n", + ")\n", "\n", - "# create the experiment object\n", - "exp = Experiment(env)\n", + "# number of time steps\n", + "flow_params['env'].horizon = 3000\n", + "exp = Experiment(flow_params)\n", "\n", - "# run the experiment for a set number of rollouts / time steps\n", - "_ = exp.run(1, 3000, convert_to_csv=True)" + "# run the sumo simulation\n", + "_ = exp.run(1, convert_to_csv=True)" ] }, { diff --git a/tutorials/tutorial02_aimsun.ipynb b/tutorials/tutorial02_aimsun.ipynb index 737955e29..91ddd2d7d 100644 --- a/tutorials/tutorial02_aimsun.ipynb +++ b/tutorials/tutorial02_aimsun.ipynb @@ -12,7 +12,7 @@ "\n", "Simulations of this form act as non-autonomous baselines and depict the behavior of human dynamics on a network. Similar simulations may also be used to evaluate the performance of hand-designed controllers on a network. This tutorial focuses primarily on the former use case, while an example of the latter may be found in `tutorial10_controllers.ipynb`.\n", "\n", - "In this exercise, we simulate an initially perturbed single lane ring road. We witness in simulation that as time advances, the initial perturbations do not dissipate, but instead propagate and expand until vehicles are forced to periodically stop and accelerate. For more information on this behavior, we refer the reader to the following article [1].\n", + "In this tutorial, we simulate an initially perturbed single lane ring road. We witness in simulation that as time advances, the initial perturbations do not dissipate, but instead propagate and expand until vehicles are forced to periodically stop and accelerate. For more information on this behavior, we refer the reader to the following article [1].\n", "\n", "## 1. Components of a Simulation\n", "All simulations, both in the presence and absence of RL, require two components: a *network*, and an *environment*. Networks describe the features of the transportation network used in simulation. This includes the positions and properties of nodes and edges constituting the lanes and junctions, as well as properties of the vehicles, traffic lights, inflows, etc. in the network. Environments, on the other hand, initialize, reset, and advance simulations, and act the primary interface between the reinforcement learning algorithm and the network. Moreover, custom environments may be used to modify the dynamical features of an network.\n", @@ -147,7 +147,7 @@ "* **resolution**: resolution of the curves on the ring. Setting this value to 1 converts the ring to a diamond.\n", "\n", "\n", - "At times, other inputs may be needed from `NetParams` to recreate proper network features/behavior. These requirements can be founded in the network's documentation. For the ring road, no attributes are needed aside from the `additional_params` terms. Furthermore, for this exercise, we use the network's default parameters when creating the `NetParams` object." + "At times, other inputs may be needed from `NetParams` to recreate proper network features/behavior. These requirements can be founded in the network's documentation. For the ring road, no attributes are needed aside from the `additional_params` terms. Furthermore, for this tutorial, we use the network's default parameters when creating the `NetParams` object." ] }, { @@ -187,7 +187,7 @@ "source": [ "### 2.5 TrafficLightParams\n", "\n", - "`TrafficLightParams` are used to desribe the positions and types of traffic lights in the network. These inputs are outside the scope of this tutorial, and instead are covered in `exercise06_traffic_lights.ipynb`. For our example, we create an empty `TrafficLightParams` object, thereby ensuring that none are placed on any nodes." + "`TrafficLightParams` are used to desribe the positions and types of traffic lights in the network. These inputs are outside the scope of this tutorial, and instead are covered in `tutorial10_traffic_lights.ipynb`. For our example, we create an empty `TrafficLightParams` object, thereby ensuring that none are placed on any nodes." ] }, { @@ -242,15 +242,26 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Although we will not be training any autonomous agents in this exercise, the use of an environment allows us to view the cumulative reward simulation rollouts receive in the absence of autonomy.\n", + "Although we will not be training any autonomous agents in this tutorial, the use of an environment allows us to view the cumulative reward simulation rollouts receive in the absence of autonomy.\n", "\n", - "Envrionments in Flow are parametrized by three components:\n", - "* `EnvParams`\n", + "Envrionments in Flow are parametrized by several components, including the following attributes:\n", + "* `sim_params`\n", + "* `env_params`\n", + "* `network`\n", + "* `net_params`\n", + "* `initial_config`\n", + "* `network`\n", + "* `simulator`\n", + "\n", + "where `sim_params`, `env_params`, and `network` the are primary parameters of an environment. For the full list of attributes, please check `class Env` in `flow/envs/base.py`.\n", + "\n", + "Aimsun envrionments in Flow are parametrized by three components:\n", "* `AimsunParams`\n", + "* `EnvParams`\n", "* `Network`\n", "\n", "### 3.1 AimsunParams\n", - "`AimsunParams` specifies simulation-specific variables. These variables include the length a simulation step (in seconds) and whether to render the GUI when running the experiment. For this example, we consider a simulation step length of 0.1s and set `render` within the simulation params to be True in order for vehicles to appear on the GUI during the simulation." + "`AimsunParams` specifies simulation-specific variables. These variables include the length a simulation step (in seconds) and whether to render the GUI when running the experiment. For this example, we set `render` within the simulation params to be True in order for vehicles to appear on the GUI during the simulation." ] }, { @@ -337,21 +348,25 @@ "metadata": {}, "outputs": [], "source": [ - "# create the network object\n", - "network = RingNetwork(name=\"ring_example\",\n", - " vehicles=vehicles,\n", - " net_params=net_params,\n", - " initial_config=initial_config,\n", - " traffic_lights=traffic_lights)\n", - "\n", - "# create the environment object\n", - "env = AccelEnv(env_params, sim_params, network, simulator='aimsun')\n", + "flow_params = dict(\n", + " exp_tag='test_network',\n", + " env_name=AccelEnv,\n", + " network=RingNetwork,\n", + " simulator='aimsun',\n", + " sim=sim_params,\n", + " env=env_params,\n", + " net=net_params,\n", + " veh=vehicles,\n", + " initial=initial_config,\n", + " tls=traffic_lights\n", + ")\n", "\n", - "# create the experiment object\n", - "exp = Experiment(env)\n", + "# number of time steps\n", + "flow_params['env'].horizon = 3000\n", + "exp = Experiment(flow_params)\n", "\n", - "# run the experiment for a set number of rollouts / time steps\n", - "_ = exp.run(1, 3000)" + "# run the Aimsun simulation\n", + "_ = exp.run(1)" ] }, { diff --git a/tutorials/tutorial03_rllib.ipynb b/tutorials/tutorial03_rllib.ipynb index 9379fa008..465fa362b 100644 --- a/tutorials/tutorial03_rllib.ipynb +++ b/tutorials/tutorial03_rllib.ipynb @@ -8,13 +8,13 @@ "\n", "This tutorial walks you through the process of running traffic simulations in Flow with trainable RLlib-powered agents. Autonomous agents will learn to maximize a certain reward over the rollouts, using the [**RLlib**](https://ray.readthedocs.io/en/latest/rllib.html) library ([citation](https://arxiv.org/abs/1712.09381)) ([installation instructions](https://flow.readthedocs.io/en/latest/flow_setup.html#optional-install-ray-rllib)). Simulations of this form will depict the propensity of RL agents to influence the traffic of a human fleet in order to make the whole fleet more efficient (for some given metrics). \n", "\n", - "In this exercise, we simulate an initially perturbed single lane ring road, where we introduce a single autonomous vehicle. We witness that, after some training, that the autonomous vehicle learns to dissipate the formation and propagation of \"phantom jams\" which form when only human driver dynamics are involved.\n", + "In this tutorial, we simulate an initially perturbed single lane ring road, where we introduce a single autonomous vehicle. We witness that, after some training, that the autonomous vehicle learns to dissipate the formation and propagation of \"phantom jams\" which form when only human driver dynamics are involved.\n", "\n", "## 1. Components of a Simulation\n", "All simulations, both in the presence and absence of RL, require two components: a *network*, and an *environment*. Networks describe the features of the transportation network used in simulation. This includes the positions and properties of nodes and edges constituting the lanes and junctions, as well as properties of the vehicles, traffic lights, inflows, etc... in the network. Environments, on the other hand, initialize, reset, and advance simulations, and act as the primary interface between the reinforcement learning algorithm and the network. Moreover, custom environments may be used to modify the dynamical features of an network. Finally, in the RL case, it is in the *environment* that the state/action spaces and the reward function are defined. \n", "\n", "## 2. Setting up a Network\n", - "Flow contains a plethora of pre-designed networks used to replicate highways, intersections, and merges in both closed and open settings. All these networks are located in flow/networks. For this exercise, which involves a single lane ring road, we will use the network `RingNetwork`.\n", + "Flow contains a plethora of pre-designed networks used to replicate highways, intersections, and merges in both closed and open settings. All these networks are located in flow/networks. For this tutorial, which involves a single lane ring road, we will use the network `RingNetwork`.\n", "\n", "### 2.1 Setting up Network Parameters\n", "\n", @@ -24,9 +24,9 @@ "* net_params\n", "* initial_config\n", "\n", - "These parameters are explained in detail in exercise 1. Moreover, all parameters excluding vehicles (covered in section 2.2) do not change from the previous exercise. Accordingly, we specify them nearly as we have before, and leave further explanations of the parameters to exercise 1.\n", + "These parameters are explained in detail in `tutorial01_sumo.ipynb`. Moreover, all parameters excluding vehicles (covered in section 2.2) do not change from the previous tutorial. Accordingly, we specify them nearly as we have before, and leave further explanations of the parameters to `tutorial01_sumo.ipynb`.\n", "\n", - "We begin by choosing the network the experiment will be trained on. We use one of Flow's builtin networks, located in `flow.networks`. A list of all available netowrks can be found by running the script below." + "We begin by choosing the network the experiment will be trained on. We use one of Flow's builtin networks, located in `flow.networks`. A list of all available networks can be found by running the script below." ] }, { @@ -95,7 +95,7 @@ "\n", "The dynamics of vehicles in the `Vehicles` class can either be depicted by sumo or by the dynamical methods located in flow/controllers. For human-driven vehicles, we use the IDM model for acceleration behavior, with exogenous gaussian acceleration noise with std 0.2 m/s2 to induce perturbations that produce stop-and-go behavior. In addition, we use the `ContinousRouter` routing controller so that the vehicles may maintain their routes closed networks.\n", "\n", - "As we have done in exercise 1, human-driven vehicles are defined in the `VehicleParams` class as follows:" + "As we have done in `tutorial01_sumo.ipynb`, human-driven vehicles are defined in the `VehicleParams` class as follows:" ] }, { @@ -162,13 +162,13 @@ "\n", "Several environments in Flow exist to train RL agents of different forms (e.g. autonomous vehicles, traffic lights) to perform a variety of different tasks. The use of an environment allows us to view the cumulative reward simulation rollouts receive, along with to specify the state/action spaces.\n", "\n", - "Envrionments in Flow are parametrized by three components:\n", - "* env_params\n", - "* sumo_params\n", - "* network\n", + "Sumo envrionments in Flow are parametrized by three components:\n", + "* `SumoParams`\n", + "* `EnvParams`\n", + "* `Network`\n", "\n", "### 3.1 SumoParams\n", - "`SumoParams` specifies simulation-specific variables. These variables include the length of any simulation step and whether to render the GUI when running the experiment. For this example, we consider a simulation step length of 0.1s and activate the GUI. \n", + "`SumoParams` specifies simulation-specific variables. These variables include the length of any simulation step and whether to render the GUI when running the experiment. For this example, we consider a simulation step length of 0.1s and deactivate the GUI. \n", "\n", "**Note** For training purposes, it is highly recommanded to deactivate the GUI in order to avoid global slow down. In such case, one just needs to specify the following: `render=False`" ] @@ -181,7 +181,7 @@ "source": [ "from flow.core.params import SumoParams\n", "\n", - "sumo_params = SumoParams(sim_step=0.1, render=False)" + "sim_params = SumoParams(sim_step=0.1, render=False)" ] }, { @@ -287,8 +287,8 @@ " network=network_name,\n", " # simulator that is used by the experiment\n", " simulator='traci',\n", - " # sumo-related parameters (see flow.core.params.SumoParams)\n", - " sim=sumo_params,\n", + " # simulation-related parameters\n", + " sim=sim_params,\n", " # environment related parameters (see flow.core.params.EnvParams)\n", " env=env_params,\n", " # network-related parameters (see flow.core.params.NetParams and\n", diff --git a/tutorials/tutorial04_visualize.ipynb b/tutorials/tutorial04_visualize.ipynb index 6ee7abd6c..25da3f322 100644 --- a/tutorials/tutorial04_visualize.ipynb +++ b/tutorials/tutorial04_visualize.ipynb @@ -13,31 +13,90 @@ "collapsed": true }, "source": [ - "This tutorial describes the process of visualizing and replaying the results of Flow experiments run using RL. The process of visualizing results breaks down into two main components:\n", + "This tutorial describes the process of visualizing the results of Flow experiments, and of replaying them. \n", "\n", - "- reward plotting\n", + "**Note:** This tutorial is only relevant if you use SUMO as a simulator. We currently do not support policy replay nor data collection when using Aimsun. The only exception is for reward plotting, which is independent on whether you have used SUMO or Aimsun during training." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "## 1. Visualization components" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "The visualization of simulation results breaks down into three main components:\n", "\n", - "- policy replay\n", + "- **reward plotting**: Visualization of the reward function is an essential step in evaluating the effectiveness and training progress of RL agents.\n", "\n", - "Note that this tutorial only talks about visualization using sumo, and not other simulators like Aimsun. " + "- **policy replay**: Flow includes tools for visualizing trained policies using SUMO's GUI. This enables more granular analysis of policies beyond their accrued reward, which in turn allows users to tweak actions, observations and rewards in order to produce some desired behavior. The visualizers also generate plots of observations and a plot of the reward function over the course of the rollout.\n", + "\n", + "- **data collection and analysis**: Any Flow experiment can output its simulation data to a CSV file, `emission.csv`, containing the contents of SUMO's built-in `emission.xml` files. This file contains various data such as the speed, position, time, fuel consumption and many other metrics for every vehicle in the network and at each time step of the simulation. Once you have generated the `emission.csv` file, you can open it and read the data it contains using Python's [csv library](https://docs.python.org/3/library/csv.html) (or using Excel)." ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "collapsed": true + }, "source": [ - "
\n", + "Visualization is different depending on which reinforcement learning library you are using, if any. Accordingly, the rest of this tutorial explains how to plot rewards, replay policies and collect data when using either no RL library, RLlib, or stable-baselines. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "**Contents:**\n", + "\n", + "[How to visualize using SUMO without training](#2.1---Using-SUMO-without-training)\n", + "\n", + "[How to visualize using SUMO with RLlib](#2.2---Using-SUMO-with-RLlib)\n", "\n", - "## Visualization with RLlib" + "[**_Example: visualize data on a ring trained using RLlib_**](#2.3---Example:-Visualize-data-on-a-ring-trained-using-RLlib)\n" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "collapsed": true + }, + "source": [ + "## 2. How to visualize" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, "source": [ - "### Plotting Reward\n", + "### 2.1 - Using SUMO without training\n", "\n", - "Similarly to how rllab handles reward plotting, RLlib supports reward visualization over the period of training using `tensorboard`. `tensorboard` takes one command-line input, `--logdir`, which is an rllib result directory (usually located within an experiment directory inside your `ray_results` directory). An example function call is below." + "_In this case, since there is no training, there is no reward to plot and no policy to replay._" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "#### Data collection and analysis\n", + "\n", + "SUMO-only experiments can generate emission CSV files seamlessly:\n", + "\n", + "First, you have to tell SUMO to generate the `emission.xml` files. You can do that by specifying `emission_path` in the simulation parameters (class `SumoParams`), which is the path where the emission files will be generated. For instance:" ] }, { @@ -46,16 +105,113 @@ "metadata": {}, "outputs": [], "source": [ - "! tensorboard --logdir /ray_results/experiment_dir/result/directory" + "from flow.core.params import SumoParams\n", + "\n", + "sim_params = SumoParams(sim_step=0.1, render=True, emission_path='data')" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "collapsed": true + }, + "source": [ + "Then, you have to tell Flow to convert these XML emission files into CSV files. To do that, pass in `convert_to_csv=True` to the `run` method of your experiment object. For instance:\n", + "\n", + "```python\n", + "exp.run(1, convert_to_csv=True)\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "When running experiments, Flow will now automatically create CSV files next to the SUMO-generated XML files." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "### 2.2 - Using SUMO with RLlib " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, "source": [ - "If you do not wish to use `tensorboard`, you can also use the `flow/visualize/plot_ray_results.py` file. It takes as arguments the path to the `progress.csv` file located inside your experiment results directory, and the name(s) of the column(s) to plot. If you do not know what the name of the columns are, simply do not put any and a list of all available columns will be displayed to you. \n", + "#### Reward plotting\n", + "\n", + "RLlib supports reward visualization over the period of the training using the `tensorboard` command. It takes one command-line parameter, `--logdir`, which is an RLlib result directory. By default, it would be located within an experiment directory inside your `~/ray_results` directory. \n", + "\n", + "An example call would look like:\n", + "\n", + "`tensorboard --logdir ~/ray_results/experiment_dir/result/directory`\n", + "\n", + "You can also run `tensorboard --logdir ~/ray_results` if you want to select more than just one experiment.\n", + "\n", + "If you do not wish to use `tensorboard`, an other way is to use our `flow/visualize/plot_ray_results.py` tool. It takes as arguments:\n", + "\n", + "- the path to the `progress.csv` file located inside your experiment results directory (`~/ray_results/...`),\n", + "- the name(s) of the column(s) you wish to plot (reward or other things).\n", + "\n", + "An example call would look like:\n", + "\n", + "`flow/visualize/plot_ray_results.py ~/ray_results/experiment_dir/result/progress.csv training/return-average training/return-min`\n", + "\n", + "If you do not know what the names of the columns are, run the command without specifying any column:\n", + "\n", + "`flow/visualize/plot_ray_results.py ~/ray_results/experiment_dir/result/progress.csv`\n", + "\n", + "and the list of all available columns will be displayed to you." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "#### Policy replay\n", + "\n", + "The tool to replay a policy trained using RLlib is located at `flow/visualize/visualizer_rllib.py`. It takes as argument, first the path to the experiment results (by default located within `~/ray_results`), and secondly the number of the checkpoint you wish to visualize (which correspond to the folder `checkpoint_` inside the experiment results directory).\n", "\n", - "Example usage:" + "An example call would look like this:\n", + "\n", + "`python flow/visualize/visualizer_rllib.py ~/ray_results/experiment_dir/result/directory 1`\n", + "\n", + "There are other optional parameters which you can learn about by running `visualizer_rllib.py --help`. " + ] + }, + { + "cell_type": "markdown", + "metadata": { + "collapsed": true + }, + "source": [ + "#### Data collection and analysis\n", + "\n", + "Simulation data can be generated the same way as it is done [without training](#2.1---Using-SUMO-without-training).\n", + "\n", + "If you need to generate simulation data after the training, you can run a policy replay as mentioned above, and add the `--gen-emission` parameter.\n", + "\n", + "An example call would look like:\n", + "\n", + "`python flow/visualize/visualizer_rllib.py ~/ray_results/experiment_dir/result/directory 1 --gen_emission`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.3 - Example: Visualize data on a ring trained using RLlib" ] }, { @@ -64,23 +220,32 @@ "metadata": {}, "outputs": [], "source": [ - "! plot_ray_results.py /ray_results/experiment_dir/progress.csv training/return-average training/return-min" + "!pwd # make sure you are in the flow/tutorials folder" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Replaying a Trained Policy" + "The folder `flow/tutorials/data/trained_ring` contains the data generated in `ray_results` after training an agent on a ring scenario for 200 iterations using RLlib (the experiment can be found in `flow/examples/rllib/stabilizing_the_ring.py`).\n", + "\n", + "Let's first have a look at what's available in the `progress.csv` file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!python ../flow/visualize/plot_ray_results.py data/trained_ring/progress.csv" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "The tool to replay a policy trained using RLlib is located in `flow/visualize/visualizer_rllib.py`. It takes as argument, first the path to the experiment results, and second the number of the checkpoint you wish to visualize. \n", - "\n", - "There are other optional parameters which you can learn about by running `visualizer_rllib.py --help`. " + "This gives us a list of everything that we can plot. Let's plot the reward and its boundaries:" ] }, { @@ -89,28 +254,37 @@ "metadata": {}, "outputs": [], "source": [ - "! python ../../flow/visualize/visualizer_rllib.py /ray_results/experiment_dir/result/directory 1" + "%matplotlib notebook\n", + "# if this doesn't display anything, try with \"%matplotlib inline\" instead\n", + "%run ../flow/visualize/plot_ray_results.py data/trained_ring/progress.csv \\\n", + "episode_reward_mean episode_reward_min episode_reward_max" ] }, { "cell_type": "markdown", - "metadata": { - "collapsed": true - }, + "metadata": {}, "source": [ - "
\n", + "We can see that the policy had already converged by the iteration 50.\n", "\n", - "## Data Collection and Analysis\n", - "Any Flow experiment can output its results to a CSV file containing the contents of SUMO's built-in `emission.xml` files, specifying speed, position, time, fuel consumption, and many other metrics for all vehicles in a network over time. \n", - "\n", - "This section describes how to generate those `emission.csv` files when replaying and analyzing a trained policy." + "Now let's see what this policy looks like. Run the following script, then click on the green arrow to run the simulation (you may have to click several times)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!python ../flow/visualize/visualizer_rllib.py data/trained_ring 200 --horizon 2000" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### RLlib" + "The RL agent is properly stabilizing the ring! \n", + "\n", + "Indeed, without an RL agent, the vehicles start forming stop-and-go waves which significantly slows down the traffic, as you can see in this simulation:" ] }, { @@ -119,23 +293,39 @@ "metadata": {}, "outputs": [], "source": [ - "# --emission_to_csv does the same as above\n", - "! python ../../flow/visualize/visualizer_rllib.py results/sample_checkpoint 1 --gen_emission" + "!python ../examples/simulate.py ring" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the trained ring folder, there is a checkpoint generated every 20 iterations. Try to run the second previous command but replace 200 by 20. On the reward plot, you can see that the reward is already quite high at iteration 20, but hasn't converged yet, so the agent will perform a little less well than at iteration 200." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "As in the rllab case, the `emission.csv` file can be found in `test_time_rollout/` and used from there." + "That's it for this example! Feel free to play around with the other scripts in `flow/visualize`. Run them with the `--help` parameter and it should tell you how to use it. Also, if you need the emission file for the trained ring, you can obtain it by running the following command:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "!python ../flow/visualize/visualizer_rllib.py data/trained_ring 200 --horizon 2000 --gen_emission" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### SUMO\n", - "SUMO-only experiments can generate emission CSV files as well, based on an argument to the `experiment.run` method. `run` takes in arguments `(num_runs, num_steps, rl_actions=None, convert_to_csv=False)`. To generate an `emission.csv` file, pass in `convert_to_csv=True` in the Python file running your SUMO experiment." + "The path where the emission file is generated will be outputted at the end of the simulation." ] } ], @@ -155,7 +345,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.6.8" }, "widgets": { "state": {}, diff --git a/tutorials/tutorial05_networks.ipynb b/tutorials/tutorial05_networks.ipynb index 585f678c2..86387a80c 100644 --- a/tutorials/tutorial05_networks.ipynb +++ b/tutorials/tutorial05_networks.ipynb @@ -8,9 +8,9 @@ "\n", "This tutorial walks you through the process of generating custom networks. Networks define the network geometry of a task, as well as the constituents of the network, e.g. vehicles, traffic lights, etc... Various networks are available in Flow, depicting a diverse set of open and closed traffic networks such as ring roads, intersections, traffic light grids, straight highway merges, and more. \n", "\n", - "In this exercise, we will recreate the ring road network, seen in the figure below.\n", + "In this tutorial, we will recreate the ring road network, seen in the figure below.\n", "\n", - "\n", + "\n", "\n", "In order to recreate this network, we will design a *network* class. This class creates the configuration files needed to produce a transportation network within the simulator. It also specifies the location of edge nodes in the network, as well as the positioning of vehicles at the start of a run.\n", "\n", @@ -370,7 +370,7 @@ "metadata": {}, "source": [ "## 3. Testing the New Network\n", - "In this section, we run a new sumo simulation using our newly generated network class. For information on running sumo experiments, see `exercise01_sumo.ipynb`.\n", + "In this section, we run a new sumo simulation using our newly generated network class. For information on running sumo experiments, see `tutorial01_sumo.ipynb`.\n", "\n", "We begin by defining some of the components needed to run a sumo experiment." ] @@ -391,7 +391,7 @@ " routing_controller=(ContinuousRouter, {}),\n", " num_vehicles=22)\n", "\n", - "sumo_params = SumoParams(sim_step=0.1, render=True)\n", + "sim_params = SumoParams(sim_step=0.1, render=True)\n", "\n", "initial_config = InitialConfig(bunching=40)" ] @@ -446,20 +446,24 @@ "source": [ "from flow.core.experiment import Experiment\n", "\n", - "network = myNetwork( # we use the newly defined network class\n", - " name=\"test_network\",\n", - " vehicles=vehicles,\n", - " net_params=net_params,\n", - " initial_config=initial_config\n", + "flow_params = dict(\n", + " exp_tag='test_network',\n", + " env_name=AccelEnv,\n", + " network=myNetwork,\n", + " simulator='traci',\n", + " sim=sim_params,\n", + " env=env_params,\n", + " net=net_params,\n", + " veh=vehicles,\n", + " initial=initial_config,\n", ")\n", "\n", - "# AccelEnv allows us to test any newly generated network quickly\n", - "env = AccelEnv(env_params, sumo_params, network)\n", + "# number of time steps\n", + "flow_params['env'].horizon = 1500\n", + "exp = Experiment(flow_params)\n", "\n", - "exp = Experiment(env)\n", - "\n", - "# run the sumo simulation for a set number of time steps\n", - "_ = exp.run(1, 1500)" + "# run the sumo simulation\n", + "_ = exp.run(1)" ] }, { @@ -486,7 +490,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.7" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/tutorials/tutorial06_osm.ipynb b/tutorials/tutorial06_osm.ipynb index cb9ae749d..8a8eb387d 100644 --- a/tutorials/tutorial06_osm.ipynb +++ b/tutorials/tutorial06_osm.ipynb @@ -91,15 +91,7 @@ "sim_params = SumoParams(render=True)\n", "initial_config = InitialConfig()\n", "vehicles = VehicleParams()\n", - "vehicles.add('human', num_vehicles=100)\n", - "\n", - "# create the network\n", - "network = Network(\n", - " name='bay_bridge',\n", - " net_params=net_params,\n", - " initial_config=initial_config,\n", - " vehicles=vehicles\n", - ")" + "vehicles.add('human', num_vehicles=100)" ] }, { @@ -115,16 +107,24 @@ "metadata": {}, "outputs": [], "source": [ - "# create the environment\n", - "env = TestEnv(\n", - " env_params=env_params,\n", - " sim_params=sim_params,\n", - " network=network\n", + "flow_params = dict(\n", + " exp_tag='bay_bridge',\n", + " env_name=TestEnv,\n", + " network=Network,\n", + " simulator='traci',\n", + " sim=sim_params,\n", + " env=env_params,\n", + " net=net_params,\n", + " veh=vehicles,\n", + " initial=initial_config,\n", ")\n", "\n", - "# run the simulation for 1000 steps\n", - "exp = Experiment(env=env)\n", - "exp.run(1, 1000)" + "# number of time steps\n", + "flow_params['env'].horizon = 1000\n", + "exp = Experiment(flow_params)\n", + "\n", + "# run the sumo simulation\n", + "_ = exp.run(1)" ] }, { @@ -256,7 +256,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### 2.3 Rerunning the SImulation\n", + "### 2.3 Rerunning the Simulation\n", "\n", "We are now ready to rerun the simulation with fully defined vehicle routes and a limited number of traversable edges. If we run the cell below, we can see the new simulation in action." ] @@ -267,24 +267,24 @@ "metadata": {}, "outputs": [], "source": [ - "# create the network\n", - "new_network = BayBridgeOSMNetwork(\n", - " name='bay_bridge',\n", - " net_params=net_params,\n", - " initial_config=new_initial_config,\n", - " vehicles=vehicles,\n", + "flow_params = dict(\n", + " exp_tag='bay_bridge',\n", + " env_name=TestEnv,\n", + " network=BayBridgeOSMNetwork,\n", + " simulator='traci',\n", + " sim=sim_params,\n", + " env=env_params,\n", + " net=net_params,\n", + " veh=vehicles,\n", + " initial=new_initial_config,\n", ")\n", "\n", - "# create the environment\n", - "env = TestEnv(\n", - " env_params=env_params,\n", - " sim_params=sim_params,\n", - " network=new_network\n", - ")\n", + "# number of time steps\n", + "flow_params['env'].horizon = 10000\n", + "exp = Experiment(flow_params)\n", "\n", - "# run the simulation for 1000 steps\n", - "exp = Experiment(env=env)\n", - "exp.run(1, 10000)" + "# run the sumo simulation\n", + "_ = exp.run(1)" ] }, { diff --git a/tutorials/tutorial07_network_templates.ipynb b/tutorials/tutorial07_network_templates.ipynb index a6aac74e8..2b35fb258 100644 --- a/tutorials/tutorial07_network_templates.ipynb +++ b/tutorials/tutorial07_network_templates.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -66,11 +66,11 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "LuST_dir = \"/home/aboudy/LuSTScenario\"" + "LuST_dir = \"/path/to/LuSTScenario\"" ] }, { @@ -86,7 +86,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -106,7 +106,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -126,7 +126,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -152,55 +152,28 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:2957: RuntimeWarning: Mean of empty slice.\n", - " out=out, **kwargs)\n", - "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:80: RuntimeWarning: invalid value encountered in double_scalars\n", - " ret = ret.dtype.type(ret / rcount)\n" - ] - }, - { - "ename": "FatalTraCIError", - "evalue": "connection closed by SUMO", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mFatalTraCIError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 16\u001b[0m \u001b[0;31m# run the simulation for 1000 steps\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 17\u001b[0m \u001b[0mexp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mExperiment\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/Documents/flow/flow/core/experiment.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, num_runs, num_steps, rl_actions, convert_to_csv)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 105\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnum_steps\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrl_actions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 107\u001b[0m vel[j] = np.mean(\n\u001b[1;32m 108\u001b[0m self.env.k.vehicle.get_speed(self.env.k.vehicle.get_ids()))\n", - "\u001b[0;32m~/Documents/flow/flow/envs/base.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, rl_actions)\u001b[0m\n\u001b[1;32m 325\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 326\u001b[0m \u001b[0;31m# advance the simulation in the simulator by one step\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 327\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msimulation\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msimulation_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 328\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 329\u001b[0m \u001b[0;31m# store new observations in the vehicles and traffic lights class\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/Documents/flow/flow/core/kernel/simulation/traci.py\u001b[0m in \u001b[0;36msimulation_step\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0msimulation_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[0;34m\"\"\"See parent class.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkernel_api\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msimulationStep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 57\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/traci/connection.py\u001b[0m in \u001b[0;36msimulationStep\u001b[0;34m(self, step)\u001b[0m\n\u001b[1;32m 273\u001b[0m self._string += struct.pack(\"!BBi\", 1 +\n\u001b[1;32m 274\u001b[0m 1 + 4, tc.CMD_SIMSTEP, step)\n\u001b[0;32m--> 275\u001b[0;31m \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sendExact\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 276\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msubscriptionResults\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_subscriptionMapping\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 277\u001b[0m \u001b[0msubscriptionResults\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/traci/connection.py\u001b[0m in \u001b[0;36m_sendExact\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 95\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_socket\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 96\u001b[0m \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_socket\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 97\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mFatalTraCIError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"connection closed by SUMO\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 98\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcommand\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_queue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 99\u001b[0m \u001b[0mprefix\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"!BBB\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mFatalTraCIError\u001b[0m: connection closed by SUMO" - ] - } - ], + "outputs": [], "source": [ - "# create the network\n", - "network = TemplateNetwork(\n", - " name=\"template\",\n", - " net_params=net_params,\n", - " initial_config=initial_config,\n", - " vehicles=vehicles\n", + "flow_params = dict(\n", + " exp_tag='template',\n", + " env_name=TestEnv,\n", + " network=TemplateNetwork,\n", + " simulator='traci',\n", + " sim=sim_params,\n", + " env=env_params,\n", + " net=net_params,\n", + " veh=vehicles,\n", + " initial=initial_config,\n", ")\n", "\n", - "# create the environment\n", - "env = TestEnv(\n", - " env_params=env_params,\n", - " sim_params=sim_params,\n", - " network=network\n", - ")\n", + "# number of time steps\n", + "flow_params['env'].horizon = 1000\n", + "exp = Experiment(flow_params)\n", "\n", - "# run the simulation for 1000 steps\n", - "exp = Experiment(env=env)\n", - "_ = exp.run(1, 1000)" + "# run the sumo simulation\n", + "_ = exp.run(1)" ] }, { @@ -216,14 +189,14 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "new_net_params = NetParams(\n", " template={\n", " # network geometry features\n", - " \"net\": os.path.join(LuST_dir, \"network/lust.net.xml\")\n", + " \"net\": os.path.join(LuST_dir, \"scenario/lust.net.xml\")\n", " }\n", ")" ] @@ -241,16 +214,16 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "new_net_params = NetParams(\n", " template={\n", " # network geometry features\n", - " \"net\": os.path.join(LuST_dir, \"network/lust.net.xml\"),\n", + " \"net\": os.path.join(LuST_dir, \"scenario/lust.net.xml\"),\n", " # features associated with the properties of drivers\n", - " \"vtype\": os.path.join(LuST_dir, \"network/vtype.add.xml\")\n", + " \"vtype\": os.path.join(LuST_dir, \"scenario/vtype.add.xml\")\n", " }\n", ")\n", "\n", @@ -269,20 +242,20 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "new_net_params = NetParams(\n", " template={\n", " # network geometry features\n", - " \"net\": os.path.join(LuST_dir, \"network/lust.net.xml\"),\n", + " \"net\": os.path.join(LuST_dir, \"scenario/lust.net.xml\"),\n", " # features associated with the properties of drivers\n", - " \"vtype\": os.path.join(LuST_dir, \"network/vtypes.add.xml\"),\n", + " \"vtype\": os.path.join(LuST_dir, \"scenario/vtypes.add.xml\"),\n", " # features associated with the routes vehicles take\n", - " \"rou\": [os.path.join(LuST_dir, \"network/DUARoutes/local.0.rou.xml\"),\n", - " os.path.join(LuST_dir, \"network/DUARoutes/local.1.rou.xml\"),\n", - " os.path.join(LuST_dir, \"network/DUARoutes/local.2.rou.xml\")]\n", + " \"rou\": [os.path.join(LuST_dir, \"scenario/DUARoutes/local.0.rou.xml\"),\n", + " os.path.join(LuST_dir, \"scenario/DUARoutes/local.1.rou.xml\"),\n", + " os.path.join(LuST_dir, \"scenario/DUARoutes/local.2.rou.xml\")]\n", " }\n", ")\n", "\n", @@ -307,23 +280,24 @@ "metadata": {}, "outputs": [], "source": [ - "# create the network\n", - "network = Network(\n", - " name=\"template\",\n", - " net_params=new_net_params,\n", - " vehicles=new_vehicles\n", + "flow_params = dict(\n", + " exp_tag='template',\n", + " env_name=TestEnv,\n", + " network=Network,\n", + " simulator='traci',\n", + " sim=sim_params,\n", + " env=env_params,\n", + " net=new_net_params,\n", + " veh=new_vehicles,\n", + " initial=initial_config,\n", ")\n", "\n", - "# create the environment\n", - "env = TestEnv(\n", - " env_params=env_params,\n", - " sim_params=sim_params,\n", - " network=network\n", - ")\n", + "# number of time steps\n", + "flow_params['env'].horizon = 100000\n", + "exp = Experiment(flow_params)\n", "\n", - "# run the simulation for 100000 steps\n", - "exp = Experiment(env=env)\n", - "_ = exp.run(1, 100000)" + "# run the sumo simulation\n", + "_ = exp.run(1)" ] }, { @@ -415,22 +389,24 @@ "metadata": {}, "outputs": [], "source": [ - "network = Network(\n", - " name=\"template\", \n", - " net_params=net_params,\n", - " initial_config=initial_config,\n", - " vehicles=vehicles\n", + "flow_params = dict(\n", + " exp_tag='template',\n", + " env_name=TestEnv,\n", + " network=Network,\n", + " simulator='aimsun',\n", + " sim=sim_params,\n", + " env=env_params,\n", + " net=net_params,\n", + " veh=vehicles,\n", + " initial=initial_config,\n", ")\n", "\n", - "env = TestEnv(\n", - " env_params, \n", - " sim_params, \n", - " network, \n", - " simulator='aimsun' \n", - ")\n", + "# number of time steps\n", + "flow_params['env'].horizon = 1000\n", + "exp = Experiment(flow_params)\n", "\n", - "exp = Experiment(env)\n", - "exp.run(1, 1000)" + "# run the sumo simulation\n", + "_ = exp.run(1)" ] } ], diff --git a/tutorials/tutorial08_environments.ipynb b/tutorials/tutorial08_environments.ipynb index b02d25613..07e12b3cb 100644 --- a/tutorials/tutorial08_environments.ipynb +++ b/tutorials/tutorial08_environments.ipynb @@ -15,12 +15,12 @@ "\n", "## 1. Creating an Environment Class\n", "\n", - "In this exercise we will create an environment in which the accelerations of a handful of vehicles in the network are specified by a single centralized agent, with the objective of the agent being to improve the average speed of all vehicle in the network. In order to create this environment, we begin by inheriting the base environment class located in *flow.envs*:" + "In this tutorial we will create an environment in which the accelerations of a handful of vehicles in the network are specified by a single centralized agent, with the objective of the agent being to improve the average speed of all vehicle in the network. In order to create this environment, we begin by inheriting the base environment class located in *flow.envs*:" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -54,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -79,7 +79,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -90,12 +90,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "In addition, `Tuple` objects (not used by this exercise) allow users to combine multiple `Box` elements together." + "In addition, `Tuple` objects (not used by this tutorial) allow users to combine multiple `Box` elements together." ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -111,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -138,7 +138,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -168,7 +168,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -199,7 +199,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -229,12 +229,12 @@ "\n", "The `compute_reward` method returns the reward associated with any given state. These value may encompass returns from values within the state space (defined in section 1.5) or may contain information provided by the environment but not immediately available within the state, as is the case in partially observable tasks (or POMDPs).\n", "\n", - "For this exercise, we choose the reward function to be the average speed of all vehicles currently in the network. In order to extract this information from the environment, we use the `get_speed` method within the Vehicle kernel class to collect the current speed of all vehicles in the network, and return the average of these speeds as the reward. This is done as follows:" + "For this tutorial, we choose the reward function to be the average speed of all vehicles currently in the network. In order to extract this information from the environment, we use the `get_speed` method within the Vehicle kernel class to collect the current speed of all vehicles in the network, and return the average of these speeds as the reward. This is done as follows:" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -266,30 +266,18 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Round 0, return: 4253.8940561797535\n", - "Average, std return: 4253.8940561797535, 0.0\n", - "Average, std speed: 2.8359293707865048, 0.0\n", - "Closing connection to TraCI and stopping simulation.\n", - "Note, this may print an error message when it closes.\n" - ] - } - ], + "outputs": [], "source": [ "from flow.controllers import IDMController, ContinuousRouter\n", "from flow.core.experiment import Experiment\n", "from flow.core.params import SumoParams, EnvParams, \\\n", " InitialConfig, NetParams\n", "from flow.core.params import VehicleParams\n", - "from flow.networks.ring import LoopNetwork, ADDITIONAL_NET_PARAMS\n", + "from flow.networks.ring import RingNetwork, ADDITIONAL_NET_PARAMS\n", "\n", - "sumo_params = SumoParams(sim_step=0.1, render=True)\n", + "sim_params = SumoParams(sim_step=0.1, render=True)\n", "\n", "vehicles = VehicleParams()\n", "vehicles.add(veh_id=\"idm\",\n", @@ -304,20 +292,24 @@ "\n", "initial_config = InitialConfig(bunching=20)\n", "\n", - "network = RingNetwork(name=\"sugiyama\",\n", - " vehicles=vehicles,\n", - " net_params=net_params,\n", - " initial_config=initial_config)\n", - "\n", - "#############################################################\n", - "######## using my new environment for the simulation ########\n", - "#############################################################\n", - "env = myEnv(env_params, sumo_params, network)\n", - "#############################################################\n", - "\n", - "exp = Experiment(env)\n", - "\n", - "_ = exp.run(1, 1500)" + "flow_params = dict(\n", + " exp_tag='ring',\n", + " env_name=myEnv, # using my new environment for the simulation\n", + " network=RingNetwork,\n", + " simulator='traci',\n", + " sim=sim_params,\n", + " env=env_params,\n", + " net=net_params,\n", + " veh=vehicles,\n", + " initial=initial_config,\n", + ")\n", + "\n", + "# number of time steps\n", + "flow_params['env'].horizon = 1500\n", + "exp = Experiment(flow_params)\n", + "\n", + "# run the sumo simulation\n", + "_ = exp.run(1)" ] }, { @@ -328,7 +320,7 @@ "\n", "Next, we wish to train this environment in the presence of the autonomous vehicle agent to reduce the formation of waves in the network, thereby pushing the performance of vehicles in the network past the above expected return.\n", "\n", - "In order for an environment to be trainable in either RLLib for rllab (as we have shown in tutorials 2 and 3), the environment must be acccessable via import from *flow.envs*. In order to do so, copy the above envrionment onto a .py and import the environment in `flow.envs.__init__.py`. You can ensure that the process was successful by running the following command:" + "The below code block may be used to train the above environment using the Proximal Policy Optimization (PPO) algorithm provided by RLlib. In order to register the environment with OpenAI gym, the environment must first be placed in a separate \".py\" file and then imported via the script below. Then, the script immediately below should function regularly." ] }, { @@ -337,106 +329,174 @@ "metadata": {}, "outputs": [], "source": [ - "# NOTE: only runs if the above procedure have been performed\n", - "from flow.envs import myEnv" + "#############################################################\n", + "####### Replace this with the environment you created #######\n", + "#############################################################\n", + "from flow.envs import AccelEnv as myEnv" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Once this is done, the below code block may be used to train the above environment using the Trust Region Policy Optimization (TRPO) algorithm provided by rllab. We do not recommend training this environment to completion within a jupyter notebook setting; however, once training is complete, visualization of the resulting policy should show that the autonomous vehicle learns to dissipate the formation and propagation of waves in the network." + "**Note**: We do not recommend training this environment to completion within a jupyter notebook setting; however, once training is complete, visualization of the resulting policy should show that the autonomous vehicle learns to dissipate the formation and propagation of waves in the network." ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ - "from rllab.envs.normalized_env import normalize\n", - "from rllab.misc.instrument import run_experiment_lite\n", - "from rllab.algos.trpo import TRPO\n", - "from rllab.baselines.linear_feature_baseline import LinearFeatureBaseline\n", - "from rllab.policies.gaussian_gru_policy import GaussianGRUPolicy\n", - "\n", - "from flow.networks.ring import RingNetwork\n", + "import json\n", + "import ray\n", + "from ray.rllib.agents.registry import get_agent_class\n", + "from ray.tune import run_experiments\n", + "from ray.tune.registry import register_env\n", + "\n", + "from flow.networks.ring import RingNetwork, ADDITIONAL_NET_PARAMS\n", + "from flow.utils.registry import make_create_env\n", + "from flow.utils.rllib import FlowParamsEncoder\n", + "from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams\n", + "from flow.core.params import VehicleParams, SumoCarFollowingParams\n", "from flow.controllers import RLController, IDMController, ContinuousRouter\n", - "from flow.core.params import VehicleParams\n", - "from flow.core.params import SumoParams, EnvParams, NetParams, InitialConfig\n", - "from rllab.envs.gym_env import GymEnv\n", "\n", + "\n", + "# time horizon of a single rollout\n", "HORIZON = 1500\n", + "# number of rollouts per training iteration\n", + "N_ROLLOUTS = 20\n", + "# number of parallel workers\n", + "N_CPUS = 2\n", "\n", "\n", - "def run_task(*_):\n", - " sumo_params = SumoParams(sim_step=0.1, render=False)\n", - "\n", - " vehicles = VehicleParams()\n", - " vehicles.add(veh_id=\"rl\",\n", - " acceleration_controller=(RLController, {}),\n", - " routing_controller=(ContinuousRouter, {}),\n", - " num_vehicles=1)\n", - " vehicles.add(veh_id=\"idm\",\n", - " acceleration_controller=(IDMController, {}),\n", - " routing_controller=(ContinuousRouter, {}),\n", - " num_vehicles=21)\n", - "\n", - " env_params = EnvParams(horizon=HORIZON,\n", - " additional_params=ADDITIONAL_ENV_PARAMS)\n", - "\n", - " additional_net_params = ADDITIONAL_NET_PARAMS.copy()\n", - " net_params = NetParams(additional_params=additional_net_params)\n", - "\n", - " initial_config = InitialConfig(bunching=20)\n", - "\n", - " network = RingNetwork(name=\"sugiyama-training\",\n", - " vehicles=vehicles,\n", - " net_params=net_params,\n", - " initial_config=initial_config)\n", - "\n", - " #######################################################\n", - " ######## using my new environment for training ########\n", - " #######################################################\n", - " env_name = \"myEnv\"\n", - " #######################################################\n", - " pass_params = (env_name, sumo_params, vehicles, env_params, net_params,\n", - " initial_config, network)\n", - "\n", - " env = GymEnv(env_name, record_video=False, register_params=pass_params)\n", - " horizon = env.horizon\n", - " env = normalize(env)\n", - "\n", - " policy = GaussianGRUPolicy(\n", - " env_spec=env.spec,\n", - " hidden_sizes=(5,),\n", - " )\n", - "\n", - " baseline = LinearFeatureBaseline(env_spec=env.spec)\n", - "\n", - " algo = TRPO(\n", - " env=env,\n", - " policy=policy,\n", - " baseline=baseline,\n", - " batch_size=30000,\n", - " max_path_length=horizon,\n", - " n_itr=500,\n", - " discount=0.999,\n", - " )\n", - " algo.train(),\n", - "\n", - "\n", - "exp_tag = \"stabilizing-the-ring\"\n", - "\n", - "for seed in [5]: # , 20, 68]:\n", - " run_experiment_lite(\n", - " run_task,\n", - " n_parallel=1,\n", - " snapshot_mode=\"all\",\n", - " seed=seed,\n", - " mode=\"local\",\n", - " exp_prefix=exp_tag,\n", - " )" + "# We place one autonomous vehicle and 22 human-driven vehicles in the network\n", + "vehicles = VehicleParams()\n", + "vehicles.add(\n", + " veh_id=\"human\",\n", + " acceleration_controller=(IDMController, {\n", + " \"noise\": 0.2\n", + " }),\n", + " car_following_params=SumoCarFollowingParams(\n", + " min_gap=0\n", + " ),\n", + " routing_controller=(ContinuousRouter, {}),\n", + " num_vehicles=21)\n", + "vehicles.add(\n", + " veh_id=\"rl\",\n", + " acceleration_controller=(RLController, {}),\n", + " routing_controller=(ContinuousRouter, {}),\n", + " num_vehicles=1)\n", + "\n", + "flow_params = dict(\n", + " # name of the experiment\n", + " exp_tag=\"stabilizing_the_ring\",\n", + "\n", + " # name of the flow environment the experiment is running on\n", + " env_name=myEnv, # <------ here we replace the environment with our new environment\n", + "\n", + " # name of the network class the experiment is running on\n", + " network=RingNetwork,\n", + "\n", + " # simulator that is used by the experiment\n", + " simulator='traci',\n", + "\n", + " # sumo-related parameters (see flow.core.params.SumoParams)\n", + " sim=SumoParams(\n", + " sim_step=0.1,\n", + " render=True,\n", + " ),\n", + "\n", + " # environment related parameters (see flow.core.params.EnvParams)\n", + " env=EnvParams(\n", + " horizon=HORIZON,\n", + " warmup_steps=750,\n", + " clip_actions=False,\n", + " additional_params={\n", + " \"target_velocity\": 20,\n", + " \"sort_vehicles\": False,\n", + " \"max_accel\": 1,\n", + " \"max_decel\": 1,\n", + " },\n", + " ),\n", + "\n", + " # network-related parameters (see flow.core.params.NetParams and the\n", + " # network's documentation or ADDITIONAL_NET_PARAMS component)\n", + " net=NetParams(\n", + " additional_params=ADDITIONAL_NET_PARAMS.copy()\n", + " ),\n", + "\n", + " # vehicles to be placed in the network at the start of a rollout (see\n", + " # flow.core.params.VehicleParams)\n", + " veh=vehicles,\n", + "\n", + " # parameters specifying the positioning of vehicles upon initialization/\n", + " # reset (see flow.core.params.InitialConfig)\n", + " initial=InitialConfig(\n", + " bunching=20,\n", + " ),\n", + ")\n", + "\n", + "\n", + "def setup_exps():\n", + " \"\"\"Return the relevant components of an RLlib experiment.\n", + "\n", + " Returns\n", + " -------\n", + " str\n", + " name of the training algorithm\n", + " str\n", + " name of the gym environment to be trained\n", + " dict\n", + " training configuration parameters\n", + " \"\"\"\n", + " alg_run = \"PPO\"\n", + "\n", + " agent_cls = get_agent_class(alg_run)\n", + " config = agent_cls._default_config.copy()\n", + " config[\"num_workers\"] = N_CPUS\n", + " config[\"train_batch_size\"] = HORIZON * N_ROLLOUTS\n", + " config[\"gamma\"] = 0.999 # discount rate\n", + " config[\"model\"].update({\"fcnet_hiddens\": [3, 3]})\n", + " config[\"use_gae\"] = True\n", + " config[\"lambda\"] = 0.97\n", + " config[\"kl_target\"] = 0.02\n", + " config[\"num_sgd_iter\"] = 10\n", + " config['clip_actions'] = False # FIXME(ev) temporary ray bug\n", + " config[\"horizon\"] = HORIZON\n", + "\n", + " # save the flow params for replay\n", + " flow_json = json.dumps(\n", + " flow_params, cls=FlowParamsEncoder, sort_keys=True, indent=4)\n", + " config['env_config']['flow_params'] = flow_json\n", + " config['env_config']['run'] = alg_run\n", + "\n", + " create_env, gym_name = make_create_env(params=flow_params, version=0)\n", + "\n", + " # Register as rllib env\n", + " register_env(gym_name, create_env)\n", + " return alg_run, gym_name, config\n", + "\n", + "\n", + "alg_run, gym_name, config = setup_exps()\n", + "ray.init(num_cpus=N_CPUS + 1)\n", + "trials = run_experiments({\n", + " flow_params[\"exp_tag\"]: {\n", + " \"run\": alg_run,\n", + " \"env\": gym_name,\n", + " \"config\": {\n", + " **config\n", + " },\n", + " \"checkpoint_freq\": 20,\n", + " \"checkpoint_at_end\": True,\n", + " \"max_failures\": 999,\n", + " \"stop\": {\n", + " \"training_iteration\": 200,\n", + " },\n", + " }\n", + "})" ] }, { @@ -463,7 +523,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.7" + "version": "3.6.8" }, "widgets": { "state": {}, diff --git a/tutorials/tutorial09_controllers.ipynb b/tutorials/tutorial09_controllers.ipynb index eb09e7c3f..0aa110d11 100644 --- a/tutorials/tutorial09_controllers.ipynb +++ b/tutorials/tutorial09_controllers.ipynb @@ -238,7 +238,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.7" + "version": "3.6.8" }, "widgets": { "state": {}, diff --git a/tutorials/tutorial10_traffic_lights.ipynb b/tutorials/tutorial10_traffic_lights.ipynb index 7930b63ab..575952fc4 100644 --- a/tutorials/tutorial10_traffic_lights.ipynb +++ b/tutorials/tutorial10_traffic_lights.ipynb @@ -15,8 +15,8 @@ "source": [ "This tutorial walks through how to add traffic lights to experiments. This tutorial will use the following files:\n", "\n", - "* Experiment script for RL version of traffic lights in grid: `examples/rllib/traffic_light_grid.py`\n", - "* Experiment script for non-RL version of traffic lights in grid: `examples/sumo/traffic_light_grid.py`\n", + "* Experiment config for RL version of traffic lights in grid: `examples/exp_configs/rl/singleagent/singleagent_traffic_light_grid.py`\n", + "* Experiment config for non-RL version of traffic lights in grid: `examples/exp_configs/non_rl/traffic_light_grid.py`\n", "* Network: `traffic_light_grid.py` (class TrafficLightGridScenario)\n", "* Environment for RL version of traffic lights in grid: (class TrafficLightGridEnv)\n", "* Environment for non-RL version of traffic lights in grid: (class AccelEnv)\n", @@ -40,7 +40,7 @@ "outputs": [], "source": [ "from flow.core.params import NetParams\n", - "from flow.networks.grid import TrafficLightGridNetwork\n", + "from flow.networks.traffic_light_grid import TrafficLightGridNetwork\n", "from flow.core.params import TrafficLightParams\n", "from flow.core.params import SumoParams, EnvParams, InitialConfig, NetParams, \\\n", " InFlows, SumoCarFollowingParams\n", @@ -79,7 +79,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -129,20 +129,13 @@ "source": [ "Once the `TrafficLightParams` class is instantiated, traffic lights can be added via the `add` function. One prerequisite of using this function is knowing the node id of any node you intend to manipulate. This information is baked into the experiment's network class, as well as the experiment's `nod.xml` file. For the experiment we are using with 2 rows and 3 columns, there are 6 nodes: \"center0\" to \"center5\". \n", "\n", - "This will be the ordering of \"centers\" in our network:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - " | | |\n", - "-3-4-5-\n", - " | | |\n", - "-0-1-2-\n", - " | | |" + "This will be the ordering of \"centers\" in our network:\n", + "\n", + " | | |\n", + " -3-4-5-\n", + " | | |\n", + " -0-1-2-\n", + " | | |" ] }, { @@ -195,8 +188,7 @@ "additional_net_params = {\"grid_array\": grid_array, \"speed_limit\": 35,\n", " \"horizontal_lanes\": 1, \"vertical_lanes\": 1,\n", " \"traffic_lights\": True}\n", - "net_params = NetParams(no_internal_links=False,\n", - " additional_params=additional_net_params)\n", + "net_params = NetParams(additional_params=additional_net_params)\n", "\n", "network = TrafficLightGridNetwork(name=\"grid\",\n", " vehicles=VehicleParams(),\n", @@ -396,12 +388,15 @@ "metadata": {}, "outputs": [], "source": [ - "# keeps track of the last time the traffic lights in an intersection were allowed to change (the last time the lights were allowed to change from a red-green state to a red-yellow state.).\n", + "# keeps track of the last time the traffic lights in an intersection were allowed to change \n", + "# (the last time the lights were allowed to change from a red-green state to a red-yellow state).\n", "self.last_change = np.zeros((self.rows * self.cols, 1))\n", - "# keeps track of the direction of the intersection (the direction that is currently being allowed to flow. 0 indicates flow from top to bottom, and 1 indicates flow from left to right.)\n", + "# keeps track of the direction of the intersection (the direction that is currently being allowed\n", + "# to flow. 0 indicates flow from top to bottom, and 1 indicates flow from left to right.)\n", "self.direction = np.zeros((self.rows * self.cols, 1))\n", - "# value of 1 indicates that the intersection is in a red-yellow state (traffic lights are red for one way (e.g. north-south), while the traffic lights for the other way (e.g. west-east) are yellow\n", - ". 0 indicates that the intersection is in a red-green state.\n", + "# value of 1 indicates that the intersection is in a red-yellow state (traffic lights are red for \n", + "# one way (e.g. north-south), while the traffic lights for the other way (e.g. west-east) are yellow.\n", + "# 0 indicates that the intersection is in a red-green state.\n", "self.currently_yellow = np.zeros((self.rows * self.cols, 1))" ] }, diff --git a/tutorials/tutorial11_inflows.ipynb b/tutorials/tutorial11_inflows.ipynb index 1a9fc18d2..a66ae0e30 100644 --- a/tutorials/tutorial11_inflows.ipynb +++ b/tutorials/tutorial11_inflows.ipynb @@ -123,11 +123,11 @@ "\n", "* `veh_type`: the type of the vehicles the inflow will create (this must match one of the types set in the `VehicleParams` object),\n", "* `edge`: the name of the edge (in the network) where the inflow will insert vehicles,\n", - "* `vehs_per_hour`: the number of vehicles entering from the edge per hour at most (it may not be achievable due to congestion and safe driving behavior).\n", + "* `vehs_per_hour`: the maximum number of vehicles entering from the edge per hour (this number may not be achievable due to congestion and safe driving behavior).\n", "\n", "More options are shown in [**section 3**](#3.-Customizing-inflows).\n", "\n", - "We begin by creating an inflow of vehicles at a rate of 2000 vehicules per hour on the main highway:" + "We begin by creating an inflow of vehicles at a rate of 2000 vehicles per hour on the main highway:" ] }, { @@ -145,7 +145,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Next, we create a second inflow of vehicles on the on-merge lane at a lower rate of 100 vehicules par hour." + "Next, we create a second inflow of vehicles on the inflow_merge lane at a lower rate of 100 vehicles per hour." ] }, { @@ -173,20 +173,6 @@ "## 2. Running simulations with inflows" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "In the next section, we will add our inflows to our network and run a simulation to see them in action." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. Running simulations with inflows" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -235,23 +221,31 @@ "from flow.envs.ring.accel import AccelEnv, ADDITIONAL_ENV_PARAMS\n", "from flow.core.experiment import Experiment\n", "\n", - "sumo_params = SumoParams(render=True,\n", + "sim_params = SumoParams(render=True,\n", " sim_step=0.2)\n", "\n", "env_params = EnvParams(additional_params=ADDITIONAL_ENV_PARAMS)\n", "\n", "initial_config = InitialConfig()\n", "\n", - "network = MergeNetwork(name=\"merge-example\",\n", - " vehicles=vehicles,\n", - " net_params=net_params,\n", - " initial_config=initial_config)\n", + "flow_params = dict(\n", + " exp_tag='merge-example',\n", + " env_name=AccelEnv,\n", + " network=MergeNetwork,\n", + " simulator='traci',\n", + " sim=sim_params,\n", + " env=env_params,\n", + " net=net_params,\n", + " veh=vehicles,\n", + " initial=initial_config,\n", + ")\n", "\n", - "env = AccelEnv(env_params, sumo_params, network)\n", + "# number of time steps\n", + "flow_params['env'].horizon = 10000\n", + "exp = Experiment(flow_params)\n", "\n", - "exp = Experiment(env)\n", - "\n", - "_ = exp.run(1, 10000)" + "# run the sumo simulation\n", + "_ = exp.run(1)" ] }, { @@ -325,7 +319,7 @@ "\n", "- `period`: this is the time in seconds between two vehicles are inserted. For example, setting this to $5$ would result in vehicles entering the network every $5$ seconds (which is effectively the same as setting `vehs_per_hour` to $720$).\n", "\n", - "_Note that all these rates are **maximum** rates, meaning that if adding vehicles at the current rate would result in vehicles between too close to each other or colliding, then the rate will automatically be reduced._\n", + "_Note that all these rates are **maximum** rates, meaning that if adding vehicles at the current rate would result in vehicles being too close to each other or colliding, then the rate will automatically be reduced._\n", "\n", "Exactly **one** of these 3 parameters should be set, no more nor less. You can choose how you would rather have your vehicles enter the network. With `vehs_per_hour` and `period` (which are proportional to each other, use whichever is more convenient to define), vehicles will enter the network equally spaced, while the vehicles will be more randomly separated if you use `probability`.\n", "\n", @@ -348,7 +342,8 @@ "- `depart_speed`: this parameter lets you specify the speed at which the vehicles will enter the network. It should be a positive `float`, in meters per second. If this speed is unsafe, the departure of the vehicles is delayed. Just like for `depart_lane`, there are other options for this parameter, which are the following strings:\n", "\n", " - `\"random\"`: vehicles enter the edge with a random speed between 0 and the speed limit on the edge. The entering speed may be adapted to ensure that a safe distance to the leading vehicle is kept\n", - " - `\"speedLimit\"`: vehicles enter the edge with the maximum speed that is allowed on this edge. If that speed is unsafe, the departure is delayed.\n", + " - `\"max\"`: vehicle speeds at insertion will be adjusted to the maximum safe speed that allows insertion at the specified time to succeed\n", + "\n", " \n", "By default, `depart_speed` is set to 0.\n", "\n", @@ -418,7 +413,8 @@ "from flow.controllers import IDMController\n", "from flow.networks import MergeNetwork\n", "from flow.networks.merge import ADDITIONAL_NET_PARAMS\n", - "from flow.envs.loop.loop_accel import AccelEnv, ADDITIONAL_ENV_PARAMS\n", + "from flow.envs.ring.accel import AccelEnv, ADDITIONAL_ENV_PARAMS\n", + "\n", "\n", "\n", "# create a vehicle type\n", @@ -437,7 +433,7 @@ " edge=\"inflow_highway\",\n", " vehs_per_hour=10000,\n", " depart_lane=\"random\",\n", - " depart_speed=\"speedLimit\",\n", + " depart_speed=\"random\",\n", " color=\"white\")\n", "\n", "# inflow for (2)\n", @@ -453,7 +449,7 @@ " edge=\"inflow_merge\",\n", " probability=0.1,\n", " depart_lane=1, # left lane\n", - " depart_speed=\"random\",\n", + " depart_speed=\"max\",\n", " begin=60, # 1 minute\n", " number=30,\n", " color=\"red\")\n", @@ -479,16 +475,24 @@ "\n", "initial_config = InitialConfig()\n", "\n", - "network = MergeNetwork(name=\"merge-example\",\n", - " vehicles=vehicles,\n", - " net_params=net_params,\n", - " initial_config=initial_config)\n", - "\n", - "env = AccelEnv(env_params, sim_params, network)\n", + "flow_params = dict(\n", + " exp_tag='merge-example',\n", + " env_name=AccelEnv,\n", + " network=MergeNetwork,\n", + " simulator='traci',\n", + " sim=sim_params,\n", + " env=env_params,\n", + " net=net_params,\n", + " veh=vehicles,\n", + " initial=initial_config,\n", + ")\n", "\n", - "exp = Experiment(env)\n", + "# number of time steps\n", + "flow_params['env'].horizon = 10000\n", + "exp = Experiment(flow_params)\n", "\n", - "_ = exp.run(1, 10000)" + "# run the sumo simulation\n", + "_ = exp.run(1)" ] }, { @@ -501,7 +505,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python [default]", + "display_name": "Python 3", "language": "python", "name": "python3" }, @@ -515,7 +519,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/tutorials/tutorial12_bottlenecks.ipynb b/tutorials/tutorial12_bottlenecks.ipynb index 2589f03c7..2481f240d 100644 --- a/tutorials/tutorial12_bottlenecks.ipynb +++ b/tutorials/tutorial12_bottlenecks.ipynb @@ -4,9 +4,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Tutorial 12: Bottlenecks Experiments\n", + "# Tutorial 12: Bottleneck Experiments\n", "\n", - "This tutorial walks you through the process of *running the bottlenecks experiments*. The bottleneck experiment, depicted in Fig. 1, is intended to illustrate the dynamics of traffic in a bottleneck. In particular, our bottleneck is intended to imitate the dynamics of traffic on the San Francisco-Oakland Bay Bridge, where fifteen lanes narrow to eight to five. In our bottleneck, we have N * 4 lanes narrow to N * 2 to N, where N is a scaling factor that can be used to increase the number of lanes. As demonstrated in Fig. 2, this bottleneck has a phenomenon known as *capacity drop*: as the number of vehicles flowing into the system increases the number of exiting vehicles initially increases. However, above a critical flow of entering vehicles, the output of the system starts to decrease as congestion forms. Just like in the San Francisco-Oakland Bay Bridge, there is a configurable toll booth and a set of traffic lights that can be used to control the inflow. Each of the merges is implemented as a zipper merge, in which the vehicles merge as late as possible. \n", + "This tutorial walks you through the process of *running the bottleneck experiments*. The bottleneck experiment, depicted in Fig. 1, is intended to illustrate the dynamics of traffic in a bottleneck. In particular, our bottleneck is intended to imitate the dynamics of traffic on the San Francisco-Oakland Bay Bridge, where fifteen lanes narrow to eight to five. In our bottleneck, we have N * 4 lanes narrow to N * 2 to N, where N is a scaling factor that can be used to increase the number of lanes. As demonstrated in Fig. 2, this bottleneck has a phenomenon known as *capacity drop*: as the number of vehicles flowing into the system increases the number of exiting vehicles initially increases. However, above a critical flow of entering vehicles, the output of the system starts to decrease as congestion forms. Just like in the San Francisco-Oakland Bay Bridge, there is a configurable toll booth and a set of traffic lights that can be used to control the inflow. Each of the merges is implemented as a zipper merge, in which the vehicles merge as late as possible. \n", "\n", "
\n", "\n", @@ -57,7 +57,12 @@ "\n", "import logging\n", "\n", - "def run_exp(flow_rate, scaling=1, disable_tb=True, disable_ramp_meter=True, n_crit=1000, feedback_coef=20):\n", + "def run_exp(flow_rate,\n", + " scaling=1,\n", + " disable_tb=True,\n", + " disable_ramp_meter=True,\n", + " n_crit=1000,\n", + " feedback_coef=20):\n", " # Set up SUMO to render the results, take a time_step of 0.5 seconds per simulation step\n", " sim_params = SumoParams(\n", " sim_step=0.5,\n", @@ -126,19 +131,25 @@ " lanes_distribution=float(\"inf\"),\n", " edges_distribution=[\"2\", \"3\", \"4\", \"5\"])\n", "\n", - " # Actually construct the network that constitutes the bottleneck.\n", - " network = BottleneckNetwork(\n", - " name=\"bay_bridge_toll\",\n", - " vehicles=vehicles,\n", - " net_params=net_params,\n", - " initial_config=initial_config,\n", - " traffic_lights=traffic_lights)\n", - "\n", - " # The environment that defines the Markov decision process of our system\n", - " env = BottleneckEnv(env_params, sim_params, network)\n", - "\n", - " \n", - " Experiment(env).run(1, 1000)" + " flow_params = dict(\n", + " exp_tag='bay_bridge_toll',\n", + " env_name=BottleneckEnv,\n", + " network=BottleneckNetwork,\n", + " simulator='traci',\n", + " sim=sim_params,\n", + " env=env_params,\n", + " net=net_params,\n", + " veh=vehicles,\n", + " initial=initial_config,\n", + " tls=traffic_lights,\n", + " )\n", + "\n", + " # number of time steps\n", + " flow_params['env'].horizon = 1000\n", + " exp = Experiment(flow_params)\n", + "\n", + " # run the sumo simulation\n", + " _ = exp.run(1)" ] }, { @@ -341,7 +352,7 @@ "\n", "While there are many different ways you might use autonomous vehicles (AVs) to try to prevent the effect of a capacity drop, here we demonstrate the particular control scheme used in \"Lagrangian control through deep-rl: Applications to bottleneck decongestion\" by E. Vinitsky, K. Parvate et. al. \n", "\n", - "The code referenced is in *examples/rllib/velocity_bottleneck.py*.\n", + "The code referenced is in *examples/exp_configs/rl/singleagent/singleagent_bottleneck.py*.\n", "\n", "We insert a flow of autonomous vehicles as a fraction of the total flow of the system. Due to randomness in the inflow, the number of AVs in the system varies.\n", "In this scheme, all of the AVs are controlled by a single controller. However, because neural network controllers necessarily take in a fixed sized input and have a fixed size output, we come up with a state parametrization and action parametrization that can handle the varying number of vehicles.\n", @@ -395,7 +406,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.5.2" + "version": "3.6.8" } }, "nbformat": 4, diff --git a/tutorials/tutorial13_rllib_ec2.ipynb b/tutorials/tutorial13_rllib_ec2.ipynb index 0bdf4c020..9bb6764d1 100644 --- a/tutorials/tutorial13_rllib_ec2.ipynb +++ b/tutorials/tutorial13_rllib_ec2.ipynb @@ -49,7 +49,7 @@ " \n", "2. Use the `ray exec` command to communicate with your cluster. \n", "\n", - " `ray exec ray_autoscale.yaml \"flow/examples/stabilizing_the_ring.py\"`\n", + " `ray exec ray_autoscale.yaml \"flow/examples/train.py singleagent_ring\"`\n", " * For a list of options you can provide to this command which will enable a variety of helpful options such as running in tmux or stopping after the command completes, view the link at the beginning of this tutorial.\n", " \n", "3. Attach to the cluster via `ray attach`.\n", @@ -61,7 +61,7 @@ "Note that the above steps 2-3 can become tedious if you create multiple clusters, and thus there are many versions of ray_autoscale.yaml lying around. For further explanation, read on: ray commands identify clusters according to the cluster_name attribute in ray_autoscale.yaml, so if you create 'test_0', test_1', 'test_2', 'test_3', and 'test_4' by simply erasing 'test_0' and replacing it with 'test_1', and so on, you would have to manually change the cluster_name in ray_autoscale.yaml to specify which cluster you intend to interact with while using `ray attach`, `ray exec`, or other `ray` commands. An alternative is this: when the cluster is created i.e. after `ray up ray_autoscale.yaml -y` is successful, it returns a ssh command to connect to that cluster's IP directly. When running multiple clusters, it can be useful to save these ssh commands.\n", "\n", "Note note, that a helpful, streamlined method of starting and executing a cluster in one fell swoop can be done via:
\n", - "4. `ray exec ray_autoscale.yaml flow/examples/stabilizing_the_ring.py --start`\n" + "4. `ray exec ray_autoscale.yaml \"flow/examples/train.py singleagent_ring\" --start`\n" ] }, { @@ -84,7 +84,7 @@ " - To scroll within the session: ctrl-b + \\[\n", " - To exit scroll mode: `q`\n", " \n", - "* Information about managing results: As usual, ray results will be automatically written to /$HOME/ray_results. To upload these results to Amazon s3, you should configure this step before running the experiment. An argument should be included within flow_params in the runner script (i.e. stabilizing_the_ring.py) in the following fashion (note the # CHANGE ME!!! comment):\n", + "* Information about managing results: As usual, ray results will be automatically written to /$HOME/ray_results. To upload these results to Amazon s3, you should configure this step before running the experiment. An argument should be included within flow_params in the runner script (i.e. singleagent_ring.py) in the following fashion (note the # CHANGE ME!!! comment):\n", "\n", "```\n", "if __name__ == \"__main__\":\n", @@ -155,4 +155,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/tutorials/tutorial14_mutiagent.ipynb b/tutorials/tutorial14_mutiagent.ipynb new file mode 100644 index 000000000..4f28e1cdf --- /dev/null +++ b/tutorials/tutorial14_mutiagent.ipynb @@ -0,0 +1,549 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tutorial 14: Multiagent\n", + "\n", + "This tutorial covers the implementation of multiagent experiments in Flow. It assumes some level of knowledge or experience in writing custom environments and running experiments with RLlib. The rest of the tutorial is organized as follows. Section 1 describes the procedure through which custom environments can be augmented to generate multiagent environments. Then, section 2 walks you through an example of running a multiagent environment\n", + "in RLlib." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Creating a Multiagent Environment Class\n", + "\n", + "In this part we will be setting up steps to create a multiagent environment. We begin by importing the abstract multi-agent evironment class." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# import the base Multi-agent environment \n", + "from flow.envs.multiagent.base import MultiEnv" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In multiagent experiments, the agent can either share a policy (\"shared policy\") or have different policies (\"non-shared policy\"). In the following subsections, we describe the two." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.1 Shared policies\n", + "In the multi-agent environment with a shared policy, different agents will use the same policy. \n", + "\n", + "We define the environment class, and inherit properties from the Multi-agent version of base env." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "class SharedMultiAgentEnv(MultiEnv):\n", + " pass" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This environment will provide the interface for running and modifying the multiagent experiment. Using this class, we are able to start the simulation (e.g. in SUMO), provide a network to specify a configuration and controllers, perform simulation steps, and reset the simulation to an initial configuration.\n", + "\n", + "For the multi-agent experiments, certain functions of the `MultiEnv` will be changed according to the agents. Some functions will be defined according to a *single* agent, while the other functions will be defined according to *all* agents.\n", + "\n", + "In the follwing functions, observation space and action space will be defined for a *single* agent (not all agents):\n", + "\n", + "* **observation_space**\n", + "* **action_space**\n", + "\n", + "For instance, in a multiagent traffic light grid, if each agents is considered as a single intersection controlling the traffic lights of the intersection, the observation space can be define as *normalized* velocities and distance to a *single* intersection for nearby vehicles, that is defined for every intersection. " + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def observation_space(self):\n", + " \"\"\"State space that is partially observed.\n", + "\n", + " Velocities and distance to intersections for nearby\n", + " vehicles ('num_observed') from each direction.\n", + " \"\"\"\n", + " tl_box = Box(\n", + " low=0.,\n", + " high=1,\n", + " shape=(2 * 4 * self.num_observed),\n", + " dtype=np.float32)\n", + " return tl_box" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The action space can be defined for a *single* intersection as follows" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "def action_space(self):\n", + " \"\"\"See class definition.\"\"\"\n", + " if self.discrete: \n", + " # each intersection is an agent, and the action is simply 0 or 1. \n", + " # - 0 means no-change in the traffic light \n", + " # - 1 means switch the direction\n", + " return Discrete(2)\n", + " else:\n", + " return Box(low=0, high=1, shape=(1,), dtype=np.float32)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Conversely, the following functions (including their return values) will be defined to take into account *all* agents:\n", + "\n", + "* **apply_rl_actions**\n", + "* **get_state**\n", + "* **compute_reward**\n", + "\n", + "Instead of calculating actions, state, and reward for a single agent, in these functions, the ctions, state, and reward will be calculated for all the agents in the system. To do so, we create a dictionary with agent ids as keys and different parameters (actions, state, and reward ) as vaules. For example, in the following `_apply_rl_actions` function, based on the action of intersections (0 or 1), the state of the intersections' traffic lights will be changed." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "class SharedMultiAgentEnv(MultiEnv): \n", + " def _apply_rl_actions(self, rl_actions):\n", + " for agent_name in rl_actions:\n", + " action = rl_actions[agent_name]\n", + " # check if the action space is discrete\n", + " \n", + " # check if our timer has exceeded the yellow phase, meaning it\n", + " # should switch to red\n", + " if self.currently_yellow[tl_num] == 1: # currently yellow\n", + " self.last_change[tl_num] += self.sim_step\n", + " if self.last_change[tl_num] >= self.min_switch_time: # check if our timer has exceeded the yellow phase, meaning it\n", + " # should switch to red\n", + " if self.direction[tl_num] == 0:\n", + " self.k.traffic_light.set_state(\n", + " node_id='center{}'.format(tl_num),\n", + " state=\"GrGr\")\n", + " else:\n", + " self.k.traffic_light.set_state(\n", + " node_id='center{}'.format(tl_num),\n", + " state='rGrG')\n", + " self.currently_yellow[tl_num] = 0\n", + " else:\n", + " if action:\n", + " if self.direction[tl_num] == 0:\n", + " self.k.traffic_light.set_state(\n", + " node_id='center{}'.format(tl_num),\n", + " state='yryr')\n", + " else:\n", + " self.k.traffic_light.set_state(\n", + " node_id='center{}'.format(tl_num),\n", + " state='ryry')\n", + " self.last_change[tl_num] = 0.0\n", + " self.direction[tl_num] = not self.direction[tl_num]\n", + " self.currently_yellow[tl_num] = 1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Similarly, the `get_state` and `compute_reward` methods support the dictionary structure and add the observation and reward, respectively, as a value for each correpsonding key, that is agent id. " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "class SharedMultiAgentEnv(MultiEnv): \n", + "\n", + " def get_state(self):\n", + " \"\"\"Observations for each intersection\n", + "\n", + " :return: dictionary which contains agent-wise observations as follows:\n", + " - For the self.num_observed number of vehicles closest and incomingsp\n", + " towards traffic light agent, gives the vehicle velocity and distance to\n", + " intersection.\n", + " \"\"\"\n", + " # Normalization factors\n", + " max_speed = max(\n", + " self.k.network.speed_limit(edge)\n", + " for edge in self.k.network.get_edge_list())\n", + " max_dist = max(grid_array[\"short_length\"], grid_array[\"long_length\"],\n", + " grid_array[\"inner_length\"])\n", + "\n", + " # Observed vehicle information\n", + " speeds = []\n", + " dist_to_intersec = []\n", + "\n", + " for _, edges in self.network.node_mapping:\n", + " local_speeds = []\n", + " local_dists_to_intersec = []\n", + " # .... More code here (removed for simplicity of example)\n", + " # ....\n", + "\n", + " speeds.append(local_speeds)\n", + " dist_to_intersec.append(local_dists_to_intersec)\n", + " \n", + " obs = {}\n", + " for agent_id in self.k.traffic_light.get_ids():\n", + " # .... More code here (removed for simplicity of example)\n", + " # ....\n", + " observation = np.array(np.concatenate(speeds, dist_to_intersec))\n", + " obs.update({agent_id: observation})\n", + " return obs\n", + "\n", + "\n", + " def compute_reward(self, rl_actions, **kwargs):\n", + " if rl_actions is None:\n", + " return {}\n", + "\n", + " if self.env_params.evaluate:\n", + " rew = -rewards.min_delay_unscaled(self)\n", + " else:\n", + " rew = -rewards.min_delay_unscaled(self) \\\n", + " + rewards.penalize_standstill(self, gain=0.2)\n", + "\n", + " # each agent receives reward normalized by number of lights\n", + " rew /= self.num_traffic_lights\n", + "\n", + " rews = {}\n", + " for rl_id in rl_actions.keys():\n", + " rews[rl_id] = rew\n", + " return rews" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.2 Non-shared policies\n", + "\n", + "In the multi-agent environment with a non-shared policy, different agents will use different policies. In what follows we will see the two agents in a ring road using two different policies, 'adversary' and 'av' (non-adversary).\n", + "\n", + "Similarly to the shared policies, observation space and action space will be defined for a *single* agent (not all agents):\n", + "\n", + "* **observation_space**\n", + "* **action_space**\n", + "\n", + "And, the following functions (including their return values) will be defined to take into account *all* agents::\n", + "\n", + "* **apply_rl_actions**\n", + "* **get_state**\n", + "* **compute_reward**\n", + "\n", + "\\* Note that, when observation space and action space will be defined for a single agent, it means that all agents should have the same dimension (i.e. space) of observation and action, even when their policise are not the same. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let us start with defining `apply_rl_actions` function. In order to make it work for a non-shared policy multi-agent ring road, we define `rl_actions` as a combinations of each policy actions plus the `perturb_weight`." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "class NonSharedMultiAgentEnv(MultiEnv):\n", + " def _apply_rl_actions(self, rl_actions):\n", + " # the names of all autonomous (RL) vehicles in the network\n", + " agent_ids = [\n", + " veh_id for veh_id in self.sorted_ids\n", + " if veh_id in self.k.vehicle.get_rl_ids()\n", + " ]\n", + " # define different actions for different multi-agents \n", + " av_action = rl_actions['av']\n", + " adv_action = rl_actions['adversary']\n", + " perturb_weight = self.env_params.additional_params['perturb_weight']\n", + " rl_action = av_action + perturb_weight * adv_action\n", + " \n", + " # use the base environment method to convert actions into accelerations for the rl vehicles\n", + " self.k.vehicle.apply_acceleration(agent_ids, rl_action)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In the `get_state` method, we define the state for each of the agents. Remember, the sate of the agents can be different. For the purpose of this example and simplicity, we define the state of the adversary and non-adversary agent to be the same. \n", + "\n", + "In the `compute_reward` method, the agents receive opposing speed rewards. The reward of the adversary agent is more when the speed of the vehicles is small, while the non-adversary agent tries to increase the speeds of the vehicles." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "class NonSharedMultiAgentEnv(MultiEnv):\n", + " def get_state(self, **kwargs):\n", + " state = np.array([[\n", + " self.k.vehicle.get_speed(veh_id) / self.k.network.max_speed(),\n", + " self.k.vehicle.get_x_by_id(veh_id) / self.k.network.length()\n", + " ] for veh_id in self.sorted_ids])\n", + " state = np.ndarray.flatten(state)\n", + " return {'av': state, 'adversary': state}\n", + "\n", + " def compute_reward(self, rl_actions, **kwargs):\n", + " if self.env_params.evaluate:\n", + " reward = np.mean(self.k.vehicle.get_speed(\n", + " self.k.vehicle.get_ids()))\n", + " return {'av': reward, 'adversary': -reward}\n", + " else:\n", + " reward = rewards.desired_velocity(self, fail=kwargs['fail'])\n", + " return {'av': reward, 'adversary': -reward}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Running Multiagent Environment in RLlib" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When running the experiment that uses a multiagent environment, we specify certain parameters in the `flow_params` dictionary. \n", + "\n", + "Similar to any other experiments, the following snippets of codes will be inserted into a blank python file (e.g. `new_multiagent_experiment.py`, and should be saved under `flow/examples/exp_configs/rl/multiagent/` directory. (all the basic imports and initialization of variables are omitted in this example for brevity)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "from flow.envs.multiagent import MultiWaveAttenuationPOEnv\n", + "from flow.networks import MultiRingNetwork\n", + "from flow.core.params import SumoParams, EnvParams, NetParams, VehicleParams, InitialConfig\n", + "from flow.controllers import ContinuousRouter, IDMController, RLController\n", + "\n", + "# time horizon of a single rollout\n", + "HORIZON = 3000\n", + "# Number of rings\n", + "NUM_RINGS = 1\n", + "\n", + "vehicles = VehicleParams()\n", + "for i in range(NUM_RINGS):\n", + " vehicles.add(\n", + " veh_id='human_{}'.format(i),\n", + " acceleration_controller=(IDMController, {\n", + " 'noise': 0.2\n", + " }),\n", + " routing_controller=(ContinuousRouter, {}),\n", + " num_vehicles=21)\n", + " vehicles.add(\n", + " veh_id='rl_{}'.format(i),\n", + " acceleration_controller=(RLController, {}),\n", + " routing_controller=(ContinuousRouter, {}),\n", + " num_vehicles=1)\n", + "\n", + "flow_params = dict(\n", + " # name of the experiment\n", + " exp_tag='multiagent_ring_road',\n", + "\n", + " # name of the flow environment the experiment is running on\n", + " env_name=MultiWaveAttenuationPOEnv,\n", + "\n", + " # name of the network class the experiment is running on\n", + " network=MultiRingNetwork,\n", + "\n", + " # simulator that is used by the experiment\n", + " simulator='traci',\n", + "\n", + " # sumo-related parameters (see flow.core.params.SumoParams)\n", + " sim=SumoParams(\n", + " sim_step=0.1,\n", + " render=False,\n", + " ),\n", + "\n", + " # environment related parameters (see flow.core.params.EnvParams)\n", + " env=EnvParams(\n", + " horizon=HORIZON,\n", + " warmup_steps=750,\n", + " additional_params={\n", + " 'max_accel': 1,\n", + " 'max_decel': 1,\n", + " 'ring_length': [230, 230],\n", + " 'target_velocity': 4\n", + " },\n", + " ),\n", + "\n", + " # network-related parameters \n", + " net=NetParams(\n", + " additional_params={\n", + " 'length': 230,\n", + " 'lanes': 1,\n", + " 'speed_limit': 30,\n", + " 'resolution': 40,\n", + " 'num_rings': NUM_RINGS\n", + " },\n", + " ),\n", + "\n", + " # vehicles to be placed in the network at the start of a rollout\n", + " veh=vehicles,\n", + "\n", + " # parameters specifying the positioning of vehicles upon initialization/\n", + " # reset\n", + " initial=InitialConfig(bunching=20.0, spacing='custom'),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then we run the following code to create the environment " + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "from flow.utils.registry import make_create_env\n", + "from ray.tune.registry import register_env\n", + "\n", + "create_env, env_name = make_create_env(params=flow_params, version=0)\n", + "\n", + "# Register as rllib env\n", + "register_env(env_name, create_env)\n", + "\n", + "test_env = create_env()\n", + "obs_space = test_env.observation_space\n", + "act_space = test_env.action_space" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.1 Shared policies\n", + "\n", + "When we run a shared-policy multiagent experiment, we refer to the same policy for each agent. In the example below the agents will use 'av' policy." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "from ray.rllib.agents.ppo.ppo_policy import PPOTFPolicy\n", + "\n", + "def gen_policy():\n", + " \"\"\"Generate a policy in RLlib.\"\"\"\n", + " return PPOTFPolicy, obs_space, act_space, {}\n", + "\n", + "\n", + "# Setup PG with an ensemble of `num_policies` different policy graphs\n", + "POLICY_GRAPHS = {'av': gen_policy()}\n", + "\n", + "\n", + "def policy_mapping_fn(_):\n", + " \"\"\"Map a policy in RLlib.\"\"\"\n", + " return 'av'\n", + "\n", + "\n", + "POLICIES_TO_TRAIN = ['av']" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 2.2 Non-shared policies\n", + "\n", + "When we run the non-shared multiagent experiment, we refer to different policies for each agent. In the example below, the policy graph will have two policies, 'adversary' and 'av' (non-adversary)." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "def gen_policy():\n", + " \"\"\"Generate a policy in RLlib.\"\"\"\n", + " return PPOTFPolicy, obs_space, act_space, {}\n", + "\n", + "\n", + "# Setup PG with an ensemble of `num_policies` different policy graphs\n", + "POLICY_GRAPHS = {'av': gen_policy(), 'adversary': gen_policy()}\n", + "\n", + "\n", + "def policy_mapping_fn(agent_id):\n", + " \"\"\"Map a policy in RLlib.\"\"\"\n", + " return agent_id" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Lastly, just like any other experiments, we run our code using `train_rllib.py` as follows:\n", + "\n", + " python flow/examples/train_rllib.py new_multiagent_experiment.py" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}