Migrate the rll_stack python packages to Python3

All Python-packages in rll_stack (except for rll_worker) can now be executed with Python3. Furthermore, they do not require a full ROS installation but can be run as a standalone application.

Nevertheless, they can still be installed as catkin packages and started via roslaunch. The Shebang-lines for these scripts has been changed to python3 which is detected by roslaunch and treated correctly.

Author: Mark Weinreuter

.ci/after_build_target_workspace.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Install dependencies required for both the Python2 test setup and the Python3 server environment
+apt install -y python3-pip
+
+pip2 install -r rll_common/requirements.txt
+pip2 install -r rll_test/requirements.txt
+
+pip3 install -r rll_server/requirements.txt
+
+# Creating docker daemon config file
+# Setting log level to fatal to prevent broken pipe during log size test
+echo -e '{
+    "storage-driver": "aufs",
+    "insecure-registries": ["rll-dockerhub.ipr.kit.edu"],
+    "log-level": "fatal"
+}' >> /etc/docker/daemon.json
+
+# Setting ros networking environment variables in bashrc
+printf "export ROS_IP=127.0.0.1\nexport ROS_MASTER_URI=http://$ROS_IP:11311\n" > ~/.bashrc
+
+# Starting docker daemon
+dockerd &
+# Starting and checking mongo daemon to give docker enough time to start
+service mongodb start
+service mongodb status
+
+# The rll_server is run with Python3 and not via roslaunch -> ROS_PACKAGE_PATH is not set
+# add rll_common, rll_server to the ROS_PACKAGE_PATH in order for rospkg to find our configs
+# add rll_common to the PYTHONPATH in order for edit_projects to import files from this module
+ROS_PACKAGE_PATH=$(pwd)/rll_common:$(pwd)/rll_server PYTHONPATH=$(pwd)/rll_common/src:${PYTHONPATH} python3 rll_common/scripts/edit_projects.py set_defaults
+
+# Checking docker daemon status
+docker info
+
+

.ci/after_script.sh

@@ -2,8 +2,17 @@
 
 set -e
 
-pip -q install flake8 
-python -m flake8 
-pip -q install 'pylint<2.0.0' 
-. /root/target_ws/install/setup.bash 
-find * -iname '*.py' | xargs python -m pylint
+pip2 -q install flake8
+pip2 -q install 'pylint<2.0.0'
+
+# Source environment to ensure the packages are found
+. /root/target_ws/install/setup.bash
+
+# rll_worker is a python2 ROS dependent package -> run pylint and flake8 here
+python2 -m flake8 rll_worker/
+find rll_worker -iname '*.py' | xargs python2 -m pylint
+
+# the rll_common can be imported under Python2 -> check the package files as well
+python2 -m flake8 rll_common/src/rll_common
+# Python3 pylint flag case a bad-option-value -> ignore those errors
+python2 -m pylint rll_common/src/rll_common -d bad-option-value

.ci/before_script.sh

@@ -32,6 +32,7 @@ build/*/MinSizeRel/*
 build/*/RelWithDebInfo/*
 build
 *.pyc
+*.egg-info
 
 *#
 *.#
@@ -44,7 +45,6 @@ build
 *.bag
 *.o
 *.md5sum
-*.pyc
 
 # Qt files
 *.pro.user
@@ -71,3 +71,10 @@ CVS/*
 /config
 *.kdev_include_paths
 *.orig
+
+.idea/
+.vscode/
+venv/
+*.log
+.mypy_reports/
+.mypy_cache/

.gitignore

@@ -8,43 +8,93 @@ image: docker:git # docker and git clients
 services:
   - docker:dind
 
-# The docker runner does not expose /tmp to the docker-in-docker service
-# This config ensures that the temp folder is located inside the project directory (e.g. for prerelease tests or SSH agent forwarding)
-variables:
-  TMPDIR: "${CI_PROJECT_DIR}.tmp"
-  UPSTREAM_WORKSPACE: ".ci.rosinstall"
-
-  ROS_DISTRO: "melodic"
-  ROS_REPO: "ros"
-
-  # rosinstall file content to ensure the rll_sdk dependency is installed
-  PROJECT_ROSINSTALL_FILENAME: ".ci.rosinstall" # Used to generate the file if it does not exist
-  ROSINSTALL_SDK_TEMPLATE: "{repositories: {rll_sdk: {type: git, url: 'https://gitlab.ipr.kit.edu/rll/rll_sdk.git', version: master}}}"
-
-before_script:
-  - apk add --update bash coreutils tar # install industrial_ci dependencies
-  - git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .industrial_ci
-  # create the .rosinstall file if it does not exist
-  - test ! -f ${PROJECT_ROSINSTALL_FILENAME} && echo ${ROSINSTALL_SDK_TEMPLATE} >> ${PROJECT_ROSINSTALL_FILENAME}
-  - sed -i 's/https\:\/\/gitlab\.ipr\.kit\.edu/https\:\/\/gitlab-ci-token\:'${CI_JOB_TOKEN}'\@gitlab\.ipr\.kit\.edu/g' ${CI_PROJECT_DIR}/${PROJECT_ROSINSTALL_FILENAME}
-  # Creating rll_stack config files from samples
-  - cp rll_common/config/common.yaml.sample rll_common/config/common.yaml
-  - cp rll_server/config/server.yaml.sample rll_server/config/server.yaml
-  - cp rll_worker/config/worker.yaml.sample rll_worker/config/worker.yaml   
-  # download config files if they do not exist
-  - test ! -f .flake8 && wget https://gitlab.ipr.kit.edu/rll/rll_sdk/raw/master/.flake8
-  - test ! -f .pylintrc && wget https://gitlab.ipr.kit.edu/rll/rll_sdk/raw/master/.pylintrc
-  
 melodic-build-tests:
   variables:
+    # The docker runner does not expose /tmp to the docker-in-docker service
+    # This config ensures that the temp folder is located inside the project directory (e.g. for prerelease tests or SSH agent forwarding)
+    TMPDIR: "${CI_PROJECT_DIR}.tmp"
+    UPSTREAM_WORKSPACE: ".ci.repos"
+    ROS_DISTRO: "melodic"
+    ROS_REPO: "ros"
+
+    # repos file content to ensure the rll_sdk dependency is installed (this is also used as the upstream workspace)
+    PROJECT_REPOS_FILENAME: ".ci.repos" # Used to generate the file if it does not exist
+    REPOS_SDK_TEMPLATE: "{repositories: {rll_sdk: {type: git, url: 'https://gitlab.ipr.kit.edu/rll/rll_sdk.git', version: master}}}"
+
     CATKIN_LINT: "pedantic"
     CATKIN_LINT_ARGS: "--ignore unknown_package --ignore unknown_depend"
+    VERBOSE_TESTS: "true"
     # -e = setting ROS_IP variable inside the container
     # -v = turning /var/lib/docker into a volume which is necessary for writing to disk while using aufs storage driver
     # --privileged is needed for running docker in docker (which we do)
     DOCKER_RUN_OPTS: "-e ROS_IP=172.18.0.2 --privileged -v aufsvol:/var/lib/docker"
-    AFTER_BUILD_TARGET_WORKSPACE: .ci/before_script.sh
+    AFTER_BUILD_TARGET_WORKSPACE: .ci/after_build_target_workspace.sh
     AFTER_SCRIPT: .ci/after_script.sh
+
+  before_script:
+    - apk add --update bash coreutils tar # install industrial_ci dependencies
+    - git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci.git .industrial_ci
+    # create the .repos file if it does not exist
+    - test ! -f ${PROJECT_REPOS_FILENAME} && echo ${REPOS_SDK_TEMPLATE} >> ${PROJECT_REPOS_FILENAME}
+    - sed -i 's/https\:\/\/gitlab\.ipr\.kit\.edu/https\:\/\/gitlab-ci-token\:'${CI_JOB_TOKEN}'\@gitlab\.ipr\.kit\.edu/g' ${CI_PROJECT_DIR}/${PROJECT_REPOS_FILENAME}
+
+    # Create rll_stack config files from samples
+    - cp rll_common/config/common.yaml.sample rll_common/config/common.yaml
+    - cp rll_server/config/server.yaml.sample rll_server/config/server.yaml
+    - cp rll_worker/config/worker.yaml.sample rll_worker/config/worker.yaml
+    - test ! -f .pylintrc && wget https://gitlab.ipr.kit.edu/rll/rll_sdk/raw/master/.pylintrc
+
+
+
+
   script: .industrial_ci/gitlab.sh
   tags:
     - docker
+
+py3_pylint:
+  before_script:
+    - apk add --no-cache --update python3 python3-dev py3-pip gcc musl-dev libffi-dev build-base 
+    - pip3 install pylint
+    - pip3 install -r rll_server/requirements.txt
+    # Install the python modules from their directories (note the trailing slash)
+    - pip3 install rll_common/ rll_server/ rll_test/
+    - test ! -f .pylintrc && wget https://gitlab.ipr.kit.edu/rll/rll_sdk/raw/master/.pylintrc -O .pylintrc
+
+  script:
+    # Run pylint for every Python3 sub-project separately to get better statistics
+    # for directories with an __init__.py you can pass the directory, otherwise specify separate files
+    # Tornado handlers set class attributes outside of the __init__ function: W0201 disables this check
+    #
+    - pylint --rcfile=.pylintrc -d W0201 rll_server/src/rll_server/ rll_server/scripts/*.py
+    - pylint --rcfile=.pylintrc rll_common/src/rll_common/ rll_common/scripts/*.py
+    - pylint --rcfile=.pylintrc rll_test/src/rll_test/ rll_test/scripts/*.py
+
+
+py3_flake8:
+  before_script:
+    - apk add --no-cache --update python3 python3-dev py3-pip musl-dev libffi-dev build-base
+    - pip3 install flake8
+    - pip3 install -r rll_server/requirements.txt
+    - pip3 install rll_common/ rll_server/ rll_test/
+    - test ! -f .flake8 && wget https://gitlab.ipr.kit.edu/rll/rll_sdk/raw/master/.flake8
+
+  script:
+    - python3 -m flake8 rll_server/
+    - python3 -m flake8 rll_common/
+    - python3 -m flake8 rll_test/
+
+mypy:
+  before_script:
+    # If you want to generate reports, you need to add the commented out dependencies for lxml
+    - apk add --no-cache --update python3 python3-dev py3-pip musl-dev libffi-dev build-base # libxml2-dev libxslt-dev
+    - pip3 install mypy # lxml
+    - pip3 install -r rll_server/requirements.txt
+    - pip3 install rll_common/ rll_server/ rll_test/ rll_worker/
+
+  allow_failure: true
+  script:
+    # Run mypy for every sub-project separately
+    - mypy rll_server/src/ rll_server/scripts/
+    - mypy rll_common/src/ rll_common/scripts/
+    - mypy rll_test/src/ rll_test/scripts/
+    - mypy --py2 rll_worker/src/ rll_worker/scripts

.gitlab-ci.yml

@@ -25,6 +25,7 @@ information on the test framework and the Docker containers.
 
 Please note that the RLL stack is only tested with ROS Kinetic and
 Melodic.
+ 
 
 1. Create a workspace and clone the core repositories:
 
@@ -45,6 +46,10 @@ Melodic.
 
    This will also install a MongoDB database for storing job, user
    and other system data.
+   
+   The `rll_server` is a Python3 application and requires some 
+   additional dependencies. Please follow the instructions in the
+   [Readme](rll_server/Readme.md) to install these dependencies.
 
 3. Build the workspace and source it:
 

README.md

@@ -0,0 +1,35 @@
+# Global options:
+[mypy]
+python_version = 3.6
+# Warn if config options are invalid
+warn_unused_configs = True
+pretty = True
+
+follow_imports = silent
+ignore_missing_imports = True
+local_partial_types = False
+check_untyped_defs = True
+strict_optional = True
+
+# not every def has to have a type comment
+allow_untyped_defs = True
+allow_untyped_globals = False
+
+# Generate reports, the html report is really helpful
+# it highlights every line and its type checking status
+# html_report = .mypy_reports
+#txt_report = .mypy_reports
+
+
+# Enable stricter options for specific modules/files
+
+# Module based options
+[mypy-rll_server.*]
+#allow_untyped_defs = False
+ignore_missing_imports = False
+follow_imports = normal
+
+[mypy-rll_worker.*]
+#allow_untyped_defs = False
+ignore_missing_imports = False
+follow_imports = normal

mypy.ini

@@ -14,5 +14,5 @@ catkin_install_python(PROGRAMS scripts/edit_projects.py scripts/jobdata_cleanup.
 
 #catkin_lint: ignore_once missing_file
 install(FILES
-  config/common.yaml config/demos.yaml config/projects.yaml
+  config/common.yaml config/demos.yaml config/logging.yaml config/projects.yaml
   DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}/config OPTIONAL)

rll_common/CMakeLists.txt

@@ -0,0 +1,33 @@
+version: 1
+
+formatters:
+  default:
+    format: "%(asctime)s %(levelname)s %(name)s: %(message)s"
+  extended:
+    format: "%(asctime)s %(levelname)s %(name)s, line:%(lineno)d:  %(message)s"
+
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: DEBUG
+    stream: "ext://sys.stdout"
+    formatter: default
+
+  file_handler:
+    class: logging.FileHandler
+    level: INFO
+    filename: "/tmp/{log_name}.log"
+    formatter: extended
+
+  error_console_handler:
+    class: logging.StreamHandler
+    stream: "ext://sys.stderr"
+    level: ERROR
+    formatter: extended
+
+root:
+  handlers: [file_handler, error_console_handler, console]
+  level: DEBUG
+  propagate: false
+
+disable_existing_loggers: false

rll_common/config/logging.yaml

@@ -1,6 +1,6 @@
-tornado<5
 docker
-motor<1.3
-bcrypt
-ansi2html
-flake8
+rospkg
+pymongo
+PyYAML
+typing
+future