RLL WebEditor

The RLL WebEditor allows users to write and visualize their code directly in the browser.

The WebEditor extends the existing rll_stack with of the following components:
1. ContainerControl: A new package which starts a Tornado app to start/stop Docker containers and upload files into these containers via a simple API. It manages a set of project images and the WebEditor utilizes the ContainerControl to spawn containers for the editor sessions. The ContainerControl app can run on a different host from the WebEditor and is a standalone app.
2. Editor backend The rll_server has been extended with new handlers and editor session management. Users start a editor session by selecting the project they want to work on and specifying whether they would like to use the text-based Python editor or the graphical Blockly editor. The WebEditor will keep track of the user session and persist the user code in the database.
3. Editor frontend The editor frontend shows a split-screen view with the editor on the left and the visualisation area on the right. The visualisation is powered by Ros3DJS which utilizes THREE.js and creates a WebGL simulation similiar to RVIZ. The frontend communicates via Websockets to the backend which relays the commands and messages into the respective Docker containers.
4. rll_worker network setup. This MR also includes changes in the way the docker networks for the rll_worker are setup/configure. **Important:** These changes have not yet been tested on the actual robots.
5. The way to create the web editor docker containers is still via scripts and should be updated to use the image_creator scripts.

The WebEditor requires a fair amount of configuration files. The configuration for supported projects are located in the `config/editor_projects` folder and can be used as a reference for new projects.

This is a squashed commit from the weinreuter:py3-webeditor branch.

Author: Mark Weinreuter

.ci/after_build_target_workspace.sh

@@ -8,7 +8,10 @@ apt install -y python3-pip
 pip2 install -r rll_common/requirements.txt
 pip2 install -r rll_test/requirements.txt
 
+
+pip3 install -r rll_common/requirements.txt
 pip3 install -r rll_server/requirements.txt
+pip3 install -r rll_test/requirements.txt
 
 # Creating docker daemon config file
 # Setting log level to fatal to prevent broken pipe during log size test
@@ -32,6 +35,10 @@ service mongodb status
 # add rll_common to the PYTHONPATH in order for edit_projects to import files from this module
 ROS_PACKAGE_PATH=$(pwd)/rll_common:$(pwd)/rll_server PYTHONPATH=$(pwd)/rll_common/src:${PYTHONPATH} python3 rll_common/scripts/edit_projects.py set_defaults
 
+ROS_PACKAGE_PATH=$(pwd)/rll_common:$(pwd)/rll_server PYTHONPATH=$(pwd)/rll_common/src:$(pwd)/rll_server/src:${PYTHONPATH} python3 rll_server/scripts/multiprocess_server.py &
+sleep 10
+kill %2
+
 # Checking docker daemon status
 docker info
 

.gitignore

@@ -74,7 +74,10 @@ CVS/*
 
 .idea/
 .vscode/
-venv/
+*venv/
 *.log
 .mypy_reports/
-.mypy_cache/
+.mypy_cache/
+
+# Prepared submission archives
+template.tar

.gitlab-ci.yml

@@ -24,6 +24,7 @@ melodic-build-tests:
     CATKIN_LINT: "pedantic"
     CATKIN_LINT_ARGS: "--ignore unknown_package --ignore unknown_depend"
     VERBOSE_TESTS: "true"
+    IMMEDIATE_TEST_OUTPUT: "true"
     # -e = setting ROS_IP variable inside the container
     # -v = turning /var/lib/docker into a volume which is necessary for writing to disk while using aufs storage driver
     # --privileged is needed for running docker in docker (which we do)
@@ -44,16 +45,13 @@ melodic-build-tests:
     - cp rll_worker/config/worker.yaml.sample rll_worker/config/worker.yaml
     - test ! -f .pylintrc && wget https://gitlab.ipr.kit.edu/rll/rll_sdk/raw/master/.pylintrc
 
-
-
-
   script: .industrial_ci/gitlab.sh
   tags:
     - docker
 
 py3_pylint:
   before_script:
-    - apk add --no-cache --update python3 python3-dev py3-pip gcc musl-dev libffi-dev build-base 
+    - apk add --no-cache --update python3 python3-dev py3-pip gcc musl-dev libffi-dev build-base curl-dev
     - pip3 install pylint
     - pip3 install -r rll_server/requirements.txt
     # Install the python modules from their directories (note the trailing slash)
@@ -65,14 +63,15 @@ py3_pylint:
     # for directories with an __init__.py you can pass the directory, otherwise specify separate files
     # Tornado handlers set class attributes outside of the __init__ function: W0201 disables this check
     #
-    - pylint --rcfile=.pylintrc -d W0201 rll_server/src/rll_server/ rll_server/scripts/*.py
-    - pylint --rcfile=.pylintrc rll_common/src/rll_common/ rll_common/scripts/*.py
-    - pylint --rcfile=.pylintrc rll_test/src/rll_test/ rll_test/scripts/*.py
+    - python3 -m pylint --rcfile=.pylintrc -d W0201 rll_server/src/rll_server/ rll_server/scripts/*.py
+    - python3 -m pylint --rcfile=.pylintrc rll_common/src/rll_common/ rll_common/scripts/*.py
+    - python3 -m pylint --rcfile=.pylintrc rll_test/src/rll_test/ rll_test/scripts/*.py
+    - python3 -m pylint --rcfile=.pylintrc -d W0201 rll_container_control/src/rll_container_control/ rll_container_control/scripts/*.py
 
 
 py3_flake8:
   before_script:
-    - apk add --no-cache --update python3 python3-dev py3-pip musl-dev libffi-dev build-base
+    - apk add --no-cache --update python3 python3-dev py3-pip musl-dev libffi-dev build-base curl-dev
     - pip3 install flake8
     - pip3 install -r rll_server/requirements.txt
     - pip3 install rll_common/ rll_server/ rll_test/
@@ -82,14 +81,17 @@ py3_flake8:
     - python3 -m flake8 rll_server/
     - python3 -m flake8 rll_common/
     - python3 -m flake8 rll_test/
+    - python3 -m flake8 rll_container_control/
 
 mypy:
   before_script:
     # If you want to generate reports, you need to add the commented out dependencies for lxml
-    - apk add --no-cache --update python3 python3-dev py3-pip musl-dev libffi-dev build-base # libxml2-dev libxslt-dev
+    - apk add --no-cache --update python3 python3-dev py3-pip musl-dev libffi-dev build-base curl-dev # libxml2-dev libxslt-dev
     - pip3 install mypy # lxml
     - pip3 install -r rll_server/requirements.txt
     - pip3 install rll_common/ rll_server/ rll_test/ rll_worker/
+    - pip3 install rll_common/ rll_server/ rll_test/ rll_worker/ rll_container_control/
+
 
   allow_failure: true
   script:
@@ -98,3 +100,4 @@ mypy:
     - mypy rll_common/src/ rll_common/scripts/
     - mypy rll_test/src/ rll_test/scripts/
     - mypy --py2 rll_worker/src/ rll_worker/scripts
+    - mypy rll_container_control/src/ rll_container_control/scripts/

rll_common/config/logging.yaml

@@ -9,15 +9,18 @@ formatters:
 handlers:
   console:
     class: logging.StreamHandler
-    level: DEBUG
+    level: INFO
     stream: "ext://sys.stdout"
     formatter: default
 
   file_handler:
-    class: logging.FileHandler
+    class: logging.handlers.RotatingFileHandler
     level: INFO
     filename: "/tmp/{log_name}.log"
     formatter: extended
+    maxBytes: 20971520 # 20MB
+    backupCount: 14
+    encoding: utf8
 
   error_console_handler:
     class: logging.StreamHandler
@@ -30,4 +33,8 @@ root:
   level: DEBUG
   propagate: false
 
+loggers:
+  tornado.access:
+    level: WARNING
+
 disable_existing_loggers: false

rll_container_control/CMakeLists.txt

@@ -0,0 +1,20 @@
+cmake_minimum_required(VERSION 3.0.2)
+project(rll_container_control)
+
+find_package(catkin REQUIRED COMPONENTS
+    rll_common
+)
+
+catkin_python_setup()
+catkin_package()
+
+install(DIRECTORY launch DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} )
+
+catkin_install_python(PROGRAMS
+  scripts/container_control.py
+  DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION})
+
+#catkin_lint: ignore_once missing_file
+install(FILES
+  config/control.yaml config/logging.yaml
+  DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}/config OPTIONAL)

rll_container_control/Readme.md

@@ -0,0 +1,72 @@
+# RLL Container Control
+
+Tornado app to control docker containers (start, stop, restart, upload files).
+
+The container manager has a set of images it can create containers for. The settings for these images is read
+from a configuration file. Each container created by the RLLContainerControl is identified by the config identifier for
+a docker image and a unique id, e.g. a user id. This allows to create multiple containers with the same settings.
+Consequently, to perform an operation you need to pass the image identifier and uid with every request.
+
+## Network configuration
+
+First create a custom bridge network which with the following options:
+
+```shell script
+ docker network create -d bridge rll-editor -o "com.docker.network.bridge.enable_icc"="0" --subnet="10.42.42.0/24" --gateway="10.42.42.1" -o "com.docker.network.bridge.name"="rll-editor"
+```
+
+This network is not internal, i. e. accessible from other computers than the host. Inter-container communication is set to disabled.
+The containers will reside in the configured subnet (chose one that does not collide with your network setup). To make the bridge
+easier to identify in the system set the internal bridge name to "rll-editor" as well. See the [Docker documentation](https://docs.docker.com/engine/reference/commandline/network_create/)
+for more details.
+
+### Limiting connections
+
+Containers within this subnet can access other hosts in the network and the internet! To restrict the container's connection options
+some custom iptable rules should be configured. Docker adds a DOCKER-USER chain to the iptables FORWARD chain and it is recommended to put custom rules there.
+See the [Docker documentation](https://docs.docker.com/network/iptables/) for more details.
+
+To limit the in- and outgoing connections the following rules are added: 
+
+```
+export RLL_CONTAINER_IP_RANGE="10.42.42.0/24"
+export RLL_SERVER_IP="YOUR_IP_HERE"
+export RLL_CONTAINER_CONTROL_IP="YOUR_IP_HERE"
+
+# Connections from the source subnet via the input interface rll-editor are dropped except for the given destination IP
+sudo iptables -I DOCKER-USER -s $RLL_CONTAINER_IP_RANGE -i "rll-editor" -j DROP
+sudo iptables -I DOCKER-USER -s $RLL_CONTAINER_IP_RANGE -i "rll-editor" -d $RLL_SERVER_IP -p tcp --dport 1025:65535 -j ACCEPT
+
+# Connetions to the destination subnet via the output interface rll-editor are dropped except for the given source IP
+sudo iptables -I DOCKER-USER -d $RLL_CONTAINER_IP_RANGE -o "rll-editor" -j DROP
+sudo iptables -I DOCKER-USER -d $RLL_CONTAINER_IP_RANGE -o "rll-editor" -s $RLL_SERVER_IP  -j ACCEPT
+```
+**NOTE** These rules are ony a best guess on my part and could be incomplete/wrong!
+
+Connections that originate in the container subnet and are send via the configured bridge ('rll-editor') are dropped except if the are meant to reach the configured destination ip.
+Note, that the port range is set to `1025:65535`. This is done because the server opens websocket connections into the containers and uses available random ports to do so.
+It might be desirable to limit this range further. For connections targeting the docker network the same restriction applies. Only connections from the server ip are allowed.
+
+There is still one problem however, the containers are still able to reach the docker host since those packets are not forwarded
+and the DOCKER-USER chain is not applied. To circumvent connections to the docker host, the following rule should be added as well:
+
+```
+# Block connections to the LOCAL docker host (cannot add this to the DOCKER-USER chain since the packets are local and not forwarded -> DOCKER-USER chain is not applied)
+sudo iptables -A INPUT -d $RLL_CONTAINER_CONTROL_IP -i "rll-editor" -j DROP
+```
+
+### Limiting resources
+
+The containers shouldn't be able to occupy to many resources and starve other users.
+Therefore, the available resources for each container are limited. The configuration for each project
+allows to specify the amount of CPUs to make available to each container. The range of available CPUs
+can also be set in the configuration. Containers are assigned a CPU-set from the pool of available CPUs.
+If no more CPUs are available multiple containers share the same CPU.
+**Note:** the containers still see all CPUs but can only access the configured CPUs.
+
+The available RAM memory is also configurable via the project settings. Containers are allowed to swap and  
+have to make do with the assigned memory.
+
+## TODO:
+- [ ] prevent concurrent container operation (set flag on OP start, clear on done)
+- [ ] Overview-Handler use Ajax requests for restart/stop buttons

rll_container_control/config/control.yaml

@@ -0,0 +1,46 @@
+# If a container is inactive for this period of time stop the container
+inactivity_seconds: 1800
+bridge_url_template: ws://%s:%d/
+control_url_template: ws://%s:%d/
+template_path: www
+cpu_range:
+  min: 0
+  max: 32
+images:
+  # the internal name of this image (project name)
+  rll_planning_project:
+    # the actual docker image tag
+    image: "rll-webeditor-rll_planning_project-melodic:latest"
+    # the base path within the container where files for this project are uploaded
+    upload_base_path: "/home/rll_user/ws/src/rll_planning_project/"
+    # restrict the container settings
+    container_settings:
+      cpus: 1
+      disk_size: "2g"
+      network: "rll-editor"
+      # Specify which ports are exposed, they will be remapped
+      ports:
+        control: 9090
+        bridge: 9091
+
+  rll_tower_of_hanoi:
+    image: "rll-webeditor_tower_of_hanoi-melodic:latest"
+    upload_base_path: "/home/rll_user/ws/src/rll_tower_of_hanoi/tower_of_hanoi_project/"
+    container_settings:
+      cpus: 2
+      disk_size: "2g"
+      network: "rll-editor"
+      ports:
+        control: 9090
+        bridge: 9091
+
+  rll_robot_playground_project:
+    upload_base_path: "/home/rll_user/ws/src/rll_robot_playground_project/"
+    image: "rll-webeditor-rll_robot_playground_project-melodic:latest"
+    container_settings:
+      cpus: 1
+      disk_size: "2g"
+      network: "rll-editor"
+      ports:
+        control: 9090
+        bridge: 9091

rll_container_control/config/logging.yaml

@@ -0,0 +1,50 @@
+version: 1
+
+formatters:
+  default:
+    format: "%(asctime)s %(levelname)s %(name)s: %(message)s"
+  extended:
+    format: "%(asctime)s %(levelname)s %(name)s, line:%(lineno)d:  %(message)s"
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: DEBUG
+    stream: "ext://sys.stdout"
+    formatter: default
+
+  file_handler:
+    class: logging.handlers.RotatingFileHandler
+    level: INFO
+    filename: "/tmp/{log_name}.log"
+    formatter: extended
+    maxBytes: 20971520 # 20MB
+    backupCount: 14
+    encoding: utf8
+
+  error_console_handler:
+    class: logging.StreamHandler
+    stream: "ext://sys.stderr"
+    level: ERROR
+    formatter: extended
+
+loggers:
+  websockets:
+    level: INFO
+    handler: [console]
+    propagate: false
+  docker:
+    level: INFO
+    handler: [console]
+    propagate: false
+  urllib3:
+    level: INFO
+    handler: [console]
+    propagate: false
+
+root:
+  handlers: [file_handler, error_console_handler, console]
+  level: DEBUG
+  propagate: false
+
+disable_existing_loggers: false

rll_container_control/launch/control.launch

@@ -0,0 +1,4 @@
+<?xml version="1.0"?>
+<launch>
+  <node name="rll_container_control" pkg="rll_container_control" type="container_control.py" output="screen"/>
+</launch>

rll_container_control/package.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0"?>
+<package format="2">
+  <name>rll_container_control</name>
+  <version>0.0.1</version>
+  <description>Control and monitor docker containers for the RLL.</description>
+
+  <maintainer email="[email protected]">Mark Weinreuter</maintainer>
+
+  <license>GPLv3+</license>
+
+  <buildtool_depend>catkin</buildtool_depend>
+  <depend>rll_common</depend>
+
+</package>