Skip to content

Commit 79ee48d

Browse files
committed
Support for client TCP listener, core pinning and client master restarts (includes refactoring, CI config)
This adds support for the new client TCP listener that was recently added to the SDK. This also means that there will not be any syncing of ROS resources from the client to the host system anymore. Core pinning for the client is added to make sure that clients only have access to specific CPU cores and are not able to block other cores. The client master is restarted after every job run. This closes #100. Some refactoring work with the help of pylint is also included. A CI config is introduced and the package configs are improved using catkin_lint, especially regarding installation of package components. See the no_client_action_srv branch for more details.
1 parent d330554 commit 79ee48d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1186
-894
lines changed

.flake8

Lines changed: 0 additions & 6 deletions
This file was deleted.

.gitlab-ci.yml

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
# The following block is needed for the shared Docker-based runner
2+
# For local runners you might want to enable the overlay driver:
3+
# https://docs.gitlab.com/ce/ci/docker/using_docker_build.html#using-the-overlayfs-driver
4+
5+
image: docker:git # docker and git clients
6+
7+
# enable docker-in-docker
8+
services:
9+
- docker:dind
10+
11+
# The docker runner does not expose /tmp to the docker-in-docker service
12+
# This config ensures that the temp folder is located inside the project directory (e.g. for prerelease tests or SSH agent forwarding)
13+
variables:
14+
TMPDIR: "${CI_PROJECT_DIR}.tmp"
15+
UPSTREAM_WORKSPACE: "file"
16+
17+
ROSINSTALL_FILENAME: ".ci.rosinstall"
18+
ROS_DISTRO: "melodic"
19+
ROS_REPO: "ros"
20+
21+
# install the rll_sdk
22+
ROSINSTALL_SDK: "- git: {local-name: rll_sdk, uri: 'https://gitlab.ipr.kit.edu/rll/rll_sdk.git', version: master}"
23+
ROSINSTALL_DEFAULT: ${ROSINSTALL_SDK}
24+
25+
26+
before_script:
27+
- apk add --update bash coreutils tar # install industrial_ci dependencies
28+
- git clone --quiet --depth 1 https://github.com/ros-industrial/industrial_ci .industrial_ci
29+
# create the .rosinstall file if it does not exist
30+
- test ! -f ${ROSINSTALL_FILENAME} && echo ${ROSINSTALL_DEFAULT} >> ${ROSINSTALL_FILENAME}
31+
- sed -i 's/https\:\/\/gitlab\.ipr\.kit\.edu/https\:\/\/gitlab-ci-token\:'${CI_JOB_TOKEN}'\@gitlab\.ipr\.kit\.edu/g' ${CI_PROJECT_DIR}/${ROSINSTALL_FILENAME}
32+
# download config files if they do not exist
33+
- test ! -f .flake8 && wget https://gitlab.ipr.kit.edu/rll/rll_sdk/raw/master/.flake8
34+
- test ! -f .pylintrc && wget https://gitlab.ipr.kit.edu/rll/rll_sdk/raw/master/.pylintrc
35+
36+
37+
melodic-build-tests:
38+
variables:
39+
CATKIN_LINT: "pedantic"
40+
CATKIN_LINT_ARGS: "--ignore unknown_package"
41+
AFTER_SCRIPT: "pip install -r rll_common/requirements.txt && pip -q install flake8 && python -m flake8 && pip -q install 'pylint<2.0.0' && . /root/catkin_ws/install/setup.bash && find * -iname '*.py' | xargs python -m pylint"
42+
# run default -> build
43+
script: .industrial_ci/gitlab.sh
44+
tags:
45+
- docker

.gitmodules

Lines changed: 0 additions & 4 deletions
This file was deleted.

rll_common/CMakeLists.txt

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,18 @@
11
cmake_minimum_required(VERSION 2.8.3)
22
project(rll_common)
33

4-
find_package(catkin REQUIRED)
4+
find_package(catkin REQUIRED COMPONENTS
5+
rospy
6+
)
57

68
catkin_python_setup()
79

810
catkin_package()
911

10-
## Mark other files for installation (e.g. launch and bag files, etc.)
12+
catkin_install_python(PROGRAMS scripts/edit_projects.py scripts/jobdata_cleanup.py
13+
DESTINATION ${CATKIN_GLOBAL_BIN_DESTINATION})
14+
15+
#catkin_lint: ignore_once missing_file
1116
install(FILES
12-
config/rll.yaml
13-
DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}
14-
)
17+
config/common.yaml config/demos.yaml config/projects.yaml
18+
DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}/config OPTIONAL)

rll_common/config/projects.yaml

Lines changed: 6 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,9 @@ project_settings:
1515
iface_project: rll_planning_project
1616
iface_node: planning_iface
1717
sync_services_to_client:
18+
- job_finished
1819
- move
1920
- check_path
20-
sync_actions_to_host:
21-
- plan_to_goal
22-
action_publisher:
23-
plan_to_goal: path_planner
2421
rll_robot_playground_project:
2522
title: "Robot Playground"
2623
git_url: https://gitlab.ipr.kit.edu/rll/rll_robot_playground_project.git
@@ -30,14 +27,11 @@ project_settings:
3027
iface_project: rll_robot_playground_project
3128
iface_node: playground_iface
3229
sync_services_to_client:
30+
- job_finished
3331
- move_random
3432
- move_lin
3533
- move_ptp
3634
- move_joints
37-
sync_actions_to_host:
38-
- move_client
39-
action_publisher:
40-
move_client: move_sender
4135
greetings:
4236
title: "Greetings"
4337
git_url: https://gitlab.ipr.kit.edu/rll/greetings_demo.git
@@ -47,11 +41,8 @@ project_settings:
4741
iface_project: rll_robot_playground_project
4842
iface_node: playground_iface
4943
sync_services_to_client:
44+
- job_finished
5045
- move_joints
51-
sync_actions_to_host:
52-
- move_client
53-
action_publisher:
54-
move_client: move_sender
5546
tower_of_hanoi:
5647
title: "Tower of Hanoi"
5748
git_url: https://gitlab.ipr.kit.edu/rll/tower_of_hanoi.git
@@ -61,13 +52,10 @@ project_settings:
6152
iface_project: tower_of_hanoi
6253
iface_node: move_iface
6354
sync_services_to_client:
55+
- job_finished
6456
- move_lin
6557
- move_ptp
6658
- pick_place
67-
sync_actions_to_host:
68-
- move_client
69-
action_publisher:
70-
move_client: tower_hanoi
7159
test_projects:
7260
title: "Test Projects"
7361
launch_client: move_sender.launch
@@ -76,10 +64,8 @@ project_settings:
7664
iface_project: rll_move
7765
iface_node: move_iface
7866
sync_services_to_client:
67+
- job_finished
7968
- move_lin
69+
- move_ptp
8070
- pick_place
8171
- move_joints
82-
sync_actions_to_host:
83-
- move_client
84-
action_publisher:
85-
move_client: test_project

rll_common/package.xml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,13 @@
22
<package format="2">
33
<name>rll_common</name>
44
<version>0.0.0</version>
5-
<description>The rll_common package</description>
5+
<description>Common code, configs for KUKA Robot Learning Lab job processing pipeline</description>
66

77
<maintainer email="[email protected]">Wolfgang Wiedmeyer</maintainer>
88
<license>GPLv3+</license>
99

1010
<buildtool_depend>catkin</buildtool_depend>
11+
<build_depend>rospy</build_depend>
1112
<exec_depend>mongodb</exec_depend>
1213

1314
</package>

rll_common/scripts/edit_projects.py

Lines changed: 15 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import argparse
2323
import sys
2424

25-
from rll_common.rll_script_helpers import EditProjectsHelpers
25+
from rll_common.rll_script_helpers import EditProjectsHelpers, catch_input
2626

2727

2828
# TODO: move this to src so that it can be loaded as a module by the
@@ -32,6 +32,11 @@
3232
# be to move the functionality in its own class
3333
class EditProjects(object):
3434
def __init__(self):
35+
# these are assigned in the helpers class
36+
self.projects_collection = None
37+
self.demos_collection = None
38+
self.rll_settings = {}
39+
3540
self.helpers = EditProjectsHelpers(self)
3641

3742
# argparse usage modelled after
@@ -46,7 +51,7 @@ def __init__(self):
4651
demo_mode Change the demo mode of a project
4752
4853
''')
49-
self.helpers.run_script_cmd(self, parser)
54+
self.helpers.run_script_cmd(parser)
5055

5156
def set_defaults(self):
5257
parser = argparse.ArgumentParser("edit_projects.py set_defaults")
@@ -60,20 +65,18 @@ def set_defaults(self):
6065
print("Editing production data")
6166

6267
if (self.projects_collection.count() != 0
63-
or self.demos_collection.count() != 0):
68+
or self.demos_collection.count() != 0):
6469
if args.production:
6570
print("collections are not empty. Type 'yes' if you "
6671
"really want to overwrite")
67-
self.helpers.catch_input()
72+
catch_input()
6873
if not args.demos:
6974
self.projects_collection.remove({})
7075
self.demos_collection.remove({})
7176

72-
if args.production:
73-
# for security, insert with demo mode enabled by default
74-
demo_mode = True
75-
else:
76-
demo_mode = False
77+
# for security, insert with demo mode enabled by default
78+
# if in production mode
79+
demo_mode = args.production
7780

7881
if not args.demos:
7982
for project in self.rll_settings["projects"]:
@@ -87,9 +90,7 @@ def set_defaults(self):
8790
else:
8891
print("only setting default settings for demos")
8992

90-
self.helpers.insert_demos(self.rll_settings["demos"],
91-
self.rll_settings["demo_settings"],
92-
self.demos_collection)
93+
self.helpers.insert_demos()
9394

9495
print("Successfully set collections to default values")
9596

@@ -111,9 +112,9 @@ def demo_mode(self):
111112
exit(1)
112113
elif args.project == "all" and args.production:
113114
print("Type 'yes' if you really want to edit all projects")
114-
self.helpers.catch_input()
115+
catch_input()
115116

116-
if args.mode == "true":
117+
if args.mode == "true": # pylint: disable=simplifiable-if-statement
117118
mode_bool = True
118119
else:
119120
mode_bool = False

rll_common/scripts/jobdata_cleanup.py

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,22 @@
1919
#
2020

2121
from __future__ import print_function
22-
import pymongo
2322
import argparse
2423
import sys
2524
from os import path
2625
from shutil import rmtree
26+
import pymongo
2727

28-
from rll_common.rll_script_helpers import ScriptHelpers
28+
from rll_common.rll_script_helpers import ScriptHelpers, catch_input
2929

3030

31+
# TODO: remove this once more functionality is added
32+
# pylint: disable=too-few-public-methods
3133
class JobDataCleanup(object):
3234
def __init__(self):
35+
self.rll_settings = {}
36+
self.jobs_collection = None
37+
3338
self.helpers = ScriptHelpers(self)
3439

3540
parser = argparse.ArgumentParser(
@@ -41,7 +46,7 @@ def __init__(self):
4146
This command also removes the jobs from the database.
4247
4348
''')
44-
self.helpers.run_script_cmd(self, parser)
49+
self.helpers.run_script_cmd(parser)
4550

4651
def user(self):
4752
# by default, keep the last 30 jobs
@@ -74,7 +79,7 @@ def user(self):
7479
if jobs.count() > del_jobs_count:
7580
print("Found outdated jobs for this user. Type 'yes' if you "
7681
"really want to delete them")
77-
self.helpers.catch_input()
82+
catch_input()
7883
else:
7984
print("found no outdated jobs for this user")
8085

rll_common/setup.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,12 @@
33
from distutils.core import setup
44
from catkin_pkg.python_setup import generate_distutils_setup
55

6-
d = generate_distutils_setup(
6+
D = generate_distutils_setup(
77
packages=[
88
'rll_common',
99
],
1010
package_dir={'': 'src'}
1111
)
1212

1313

14-
setup(**d)
14+
setup(**D)

rll_common/src/rll_common/rll_helpers.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919
#
2020

2121
import logging
22-
import yaml
23-
import sys
2422
from os import path
23+
import sys
24+
import yaml
2525

2626
import rospkg
2727

0 commit comments

Comments
 (0)