Support for client TCP listener, core pinning and client master restarts (includes refactoring, CI config)

This adds support for the new client TCP listener that was recently added to the SDK.
This also means that there will not be any syncing of ROS resources from the client
to the host system anymore.

Core pinning for the client is added to make sure that clients only have access to
specific CPU cores and are not able to block other cores.

The client master is restarted after every job run. This closes #100.

Some refactoring work with the help of pylint is also included. A CI config is
introduced and the package configs are improved using catkin_lint, especially
regarding installation of package components.

See the no_client_action_srv branch for more details.

Author: wiewo

.flake8

@@ -0,0 +1,45 @@
+# The following block is needed for the shared Docker-based runner
+# For local runners you might want to enable the overlay driver:
+# https://docs.gitlab.com/ce/ci/docker/using_docker_build.html#using-the-overlayfs-driver
+
+image: docker:git # docker and git clients
+
+# enable docker-in-docker
+services:
+  - docker:dind
+
+# The docker runner does not expose /tmp to the docker-in-docker service
+# This config ensures that the temp folder is located inside the project directory (e.g. for prerelease tests or SSH agent forwarding)
+variables:
+  TMPDIR: "${CI_PROJECT_DIR}.tmp"
+  UPSTREAM_WORKSPACE: "file"
+
+  ROSINSTALL_FILENAME: ".ci.rosinstall"
+  ROS_DISTRO: "melodic"
+  ROS_REPO: "ros"
+
+  # install the rll_sdk
+  ROSINSTALL_SDK: "- git: {local-name: rll_sdk, uri: 'https://gitlab.ipr.kit.edu/rll/rll_sdk.git', version: master}"
+  ROSINSTALL_DEFAULT: ${ROSINSTALL_SDK}
+
+
+before_script:
+  - apk add --update bash coreutils tar # install industrial_ci dependencies
+  - git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci .industrial_ci
+  # create the .rosinstall file if it does not exist
+  - test ! -f ${ROSINSTALL_FILENAME} && echo ${ROSINSTALL_DEFAULT} >> ${ROSINSTALL_FILENAME}
+  - sed -i 's/https\:\/\/gitlab\.ipr\.kit\.edu/https\:\/\/gitlab-ci-token\:'${CI_JOB_TOKEN}'\@gitlab\.ipr\.kit\.edu/g' ${CI_PROJECT_DIR}/${ROSINSTALL_FILENAME}
+  # download config files if they do not exist
+  - test ! -f .flake8 && wget https://gitlab.ipr.kit.edu/rll/rll_sdk/raw/master/.flake8
+  - test ! -f .pylintrc && wget https://gitlab.ipr.kit.edu/rll/rll_sdk/raw/master/.pylintrc
+
+
+melodic-build-tests:
+  variables:
+    CATKIN_LINT: "pedantic"
+    CATKIN_LINT_ARGS: "--ignore unknown_package"
+    AFTER_SCRIPT: "pip install -r rll_common/requirements.txt && pip -q install flake8 && python -m flake8 && pip -q install 'pylint<2.0.0' && . /root/catkin_ws/install/setup.bash && find * -iname '*.py' | xargs python -m pylint"
+  # run default -> build
+  script: .industrial_ci/gitlab.sh
+  tags:
+    - docker

.gitlab-ci.yml

@@ -1,14 +1,18 @@
 cmake_minimum_required(VERSION 2.8.3)
 project(rll_common)
 
-find_package(catkin REQUIRED)
+find_package(catkin REQUIRED COMPONENTS
+  rospy
+)
 
 catkin_python_setup()
 
 catkin_package()
 
-## Mark other files for installation (e.g. launch and bag files, etc.)
+catkin_install_python(PROGRAMS scripts/edit_projects.py scripts/jobdata_cleanup.py
+  DESTINATION ${CATKIN_GLOBAL_BIN_DESTINATION})
+
+#catkin_lint: ignore_once missing_file
 install(FILES
-  config/rll.yaml
-  DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}
-)
+  config/common.yaml config/demos.yaml config/projects.yaml
+  DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}/config OPTIONAL)

.gitmodules

@@ -15,12 +15,9 @@ project_settings:
     iface_project: rll_planning_project
     iface_node: planning_iface
     sync_services_to_client:
+        - job_finished
         - move
         - check_path
-    sync_actions_to_host:
-        - plan_to_goal
-    action_publisher:
-        plan_to_goal: path_planner
   rll_robot_playground_project:
     title: "Robot Playground"
     git_url: https://gitlab.ipr.kit.edu/rll/rll_robot_playground_project.git
@@ -30,14 +27,11 @@ project_settings:
     iface_project: rll_robot_playground_project
     iface_node: playground_iface
     sync_services_to_client:
+        - job_finished
         - move_random
         - move_lin
         - move_ptp
         - move_joints
-    sync_actions_to_host:
-        - move_client
-    action_publisher:
-        move_client: move_sender
   greetings:
     title: "Greetings"
     git_url: https://gitlab.ipr.kit.edu/rll/greetings_demo.git
@@ -47,11 +41,8 @@ project_settings:
     iface_project: rll_robot_playground_project
     iface_node: playground_iface
     sync_services_to_client:
+        - job_finished
         - move_joints
-    sync_actions_to_host:
-        - move_client
-    action_publisher:
-        move_client: move_sender
   tower_of_hanoi:
     title: "Tower of Hanoi"
     git_url: https://gitlab.ipr.kit.edu/rll/tower_of_hanoi.git
@@ -61,13 +52,10 @@ project_settings:
     iface_project: tower_of_hanoi
     iface_node: move_iface
     sync_services_to_client:
+        - job_finished
         - move_lin
         - move_ptp
         - pick_place
-    sync_actions_to_host:
-        - move_client
-    action_publisher:
-        move_client: tower_hanoi
   test_projects:
     title: "Test Projects"
     launch_client: move_sender.launch
@@ -76,10 +64,8 @@ project_settings:
     iface_project: rll_move
     iface_node: move_iface
     sync_services_to_client:
+        - job_finished
         - move_lin
+        - move_ptp
         - pick_place
         - move_joints
-    sync_actions_to_host:
-        - move_client
-    action_publisher:
-        move_client: test_project

rll_common/CMakeLists.txt

@@ -2,12 +2,13 @@
 <package format="2">
   <name>rll_common</name>
   <version>0.0.0</version>
-  <description>The rll_common package</description>
+  <description>Common code, configs for KUKA Robot Learning Lab job processing pipeline</description>
 
   <maintainer email="[email protected]">Wolfgang Wiedmeyer</maintainer>
   <license>GPLv3+</license>
 
   <buildtool_depend>catkin</buildtool_depend>
+  <build_depend>rospy</build_depend>
   <exec_depend>mongodb</exec_depend>
 
 </package>

rll_common/config/projects.yaml

@@ -22,7 +22,7 @@
 import argparse
 import sys
 
-from rll_common.rll_script_helpers import EditProjectsHelpers
+from rll_common.rll_script_helpers import EditProjectsHelpers, catch_input
 
 
 # TODO: move this to src so that it can be loaded as a module by the
@@ -32,6 +32,11 @@
 #       be to move the functionality in its own class
 class EditProjects(object):
     def __init__(self):
+        # these are assigned in the helpers class
+        self.projects_collection = None
+        self.demos_collection = None
+        self.rll_settings = {}
+
         self.helpers = EditProjectsHelpers(self)
 
         # argparse usage modelled after
@@ -46,7 +51,7 @@ def __init__(self):
    demo_mode        Change the demo mode of a project
 
 ''')
-        self.helpers.run_script_cmd(self, parser)
+        self.helpers.run_script_cmd(parser)
 
     def set_defaults(self):
         parser = argparse.ArgumentParser("edit_projects.py set_defaults")
@@ -60,20 +65,18 @@ def set_defaults(self):
             print("Editing production data")
 
         if (self.projects_collection.count() != 0
-           or self.demos_collection.count() != 0):
+                or self.demos_collection.count() != 0):
             if args.production:
                 print("collections are not empty. Type 'yes' if you "
                       "really want to overwrite")
-                self.helpers.catch_input()
+                catch_input()
             if not args.demos:
                 self.projects_collection.remove({})
             self.demos_collection.remove({})
 
-        if args.production:
-            # for security, insert with demo mode enabled by default
-            demo_mode = True
-        else:
-            demo_mode = False
+        # for security, insert with demo mode enabled by default
+        # if in production mode
+        demo_mode = args.production
 
         if not args.demos:
             for project in self.rll_settings["projects"]:
@@ -87,9 +90,7 @@ def set_defaults(self):
         else:
             print("only setting default settings for demos")
 
-        self.helpers.insert_demos(self.rll_settings["demos"],
-                                  self.rll_settings["demo_settings"],
-                                  self.demos_collection)
+        self.helpers.insert_demos()
 
         print("Successfully set collections to default values")
 
@@ -111,9 +112,9 @@ def demo_mode(self):
             exit(1)
         elif args.project == "all" and args.production:
             print("Type 'yes' if you really want to edit all projects")
-            self.helpers.catch_input()
+            catch_input()
 
-        if args.mode == "true":
+        if args.mode == "true":  # pylint: disable=simplifiable-if-statement
             mode_bool = True
         else:
             mode_bool = False

rll_common/package.xml

@@ -19,17 +19,22 @@
 #
 
 from __future__ import print_function
-import pymongo
 import argparse
 import sys
 from os import path
 from shutil import rmtree
+import pymongo
 
-from rll_common.rll_script_helpers import ScriptHelpers
+from rll_common.rll_script_helpers import ScriptHelpers, catch_input
 
 
+# TODO: remove this once more functionality is added
+# pylint: disable=too-few-public-methods
 class JobDataCleanup(object):
     def __init__(self):
+        self.rll_settings = {}
+        self.jobs_collection = None
+
         self.helpers = ScriptHelpers(self)
 
         parser = argparse.ArgumentParser(
@@ -41,7 +46,7 @@ def __init__(self):
                This command also removes the jobs from the database.
 
 ''')
-        self.helpers.run_script_cmd(self, parser)
+        self.helpers.run_script_cmd(parser)
 
     def user(self):
         # by default, keep the last 30 jobs
@@ -74,7 +79,7 @@ def user(self):
         if jobs.count() > del_jobs_count:
             print("Found outdated jobs for this user. Type 'yes' if you "
                   "really want to delete them")
-            self.helpers.catch_input()
+            catch_input()
         else:
             print("found no outdated jobs for this user")
 

rll_common/scripts/edit_projects.py

@@ -3,12 +3,12 @@
 from distutils.core import setup
 from catkin_pkg.python_setup import generate_distutils_setup
 
-d = generate_distutils_setup(
+D = generate_distutils_setup(
     packages=[
         'rll_common',
     ],
     package_dir={'': 'src'}
 )
 
 
-setup(**d)
+setup(**D)

rll_common/scripts/jobdata_cleanup.py

@@ -19,9 +19,9 @@
 #
 
 import logging
-import yaml
-import sys
 from os import path
+import sys
+import yaml
 
 import rospkg