From afd9e09ba77cd9061c587b839dc18f14eaf54571 Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Fri, 29 Mar 2024 06:16:13 -0400 Subject: [PATCH 01/11] ci.yaml: Add workflow_dispatch event trigger Signed-off-by: Rob Woolley --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fba3e7c9..fb435542 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,7 @@ name: superflore-ci on: + workflow_dispatch: push: branches: ['master'] pull_request: From 9e373983bf9680250b64f3ca7ded944566ea33dc Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Fri, 29 Mar 2024 06:22:35 -0400 Subject: [PATCH 02/11] ci.yaml: Update os and python matrix This prevents trailing zeroes from being removed by making the values strings not numbers. Signed-off-by: Rob Woolley --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fb435542..c86fe6bc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,8 +10,8 @@ jobs: build: strategy: matrix: - os: [ubuntu-18.04, ubuntu-20.04] - python: [3.6, 3.7, 3.8, 3.9] + os: ["ubuntu-18.04", "ubuntu-20.04", "ubuntu-22.04"] + python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] name: superflore tests runs-on: ${{matrix.os}} steps: From 622851f32ef958474c1cc156cd915ae6cc1e44fd Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Fri, 29 Mar 2024 06:41:17 -0400 Subject: [PATCH 03/11] requirements.txt: Use pynose Use pynose for Python 3.11+ support Signed-off-by: Rob Woolley --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a50e25ed..282bb8fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -nose +pynose flake8 flake8-import-order termcolor From b02aec0d72f5670cc53a3e5d10017139526b0583 Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Fri, 29 Mar 2024 06:50:42 -0400 Subject: [PATCH 04/11] ci.yaml: Exclude some invalid tests Excluding tests for docker and for creating invalid directory paths. These tests do not seem relevant to testing superflore in a containerized CI/CD pipeline. Signed-off-by: Rob Woolley --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c86fe6bc..d9b6d592 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -33,7 +33,7 @@ jobs: run: | sudo rosdep init rosdep update - python -m 'nose' --exclude test_pull --exclude test_run --exclude test_logger_output + python -m 'nose' --exclude test_pull --exclude test_run --exclude test_logger_output --exclude test_docker.py --exclude test_failed_to_create python -m 'flake8' superflore --import-order-style=google From 69cde4d3afae7e53a759caecb7da287d7a5bb6c4 Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Fri, 29 Mar 2024 06:57:01 -0400 Subject: [PATCH 05/11] ci.yaml: Update to latest GitHub Actions Update the checkout and setup-python actions to the latest versions. Remove Ubuntu 18.04 as a runner image as it has been deprecated by GitHub Actions and no longer has runners available. Signed-off-by: Rob Woolley --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d9b6d592..b46c0302 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -10,14 +10,14 @@ jobs: build: strategy: matrix: - os: ["ubuntu-18.04", "ubuntu-20.04", "ubuntu-22.04"] + os: ["ubuntu-20.04", "ubuntu-22.04"] python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] name: superflore tests runs-on: ${{matrix.os}} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{matrix.python}} - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: ${{matrix.python}} - name: Install dependencies From 3a79369724e8edea08fff3b4571d58af959e4df5 Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Mon, 25 Nov 2024 16:14:34 -0800 Subject: [PATCH 06/11] ci.yaml: Update steps for ROS 2 These steps follow the installation instructions from the Rolling Ridley documentation for setting up a new system. These changes are necessary to use the ros2 package repository for newer versions of Ubuntu. https://docs.ros.org/en/rolling/Installation/Ubuntu-Install-Debs.html Signed-off-by: Rob Woolley --- .github/workflows/ci.yaml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b46c0302..e73aceaf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,9 +22,21 @@ jobs: python-version: ${{matrix.python}} - name: Install dependencies run: | - echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/ros-latest.list - wget https://raw.githubusercontent.com/ros/rosdistro/master/ros.asc -O /tmp/ros.asc - sudo apt-key add /tmp/ros.asc + echo "Set locale" + sudo apt update && sudo apt install locales + sudo locale-gen en_US en_US.UTF-8 + sudo update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 + export LANG=en_US.UTF-8 + + echo "Enable required repositories" + sudo apt install software-properties-common + sudo add-apt-repository universe + + sudo apt update && sudo apt install curl -y + sudo curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg + + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(. /etc/os-release && echo $UBUNTU_CODENAME) main" | sudo tee /etc/apt/sources.list.d/ros2.list > /dev/null + sudo apt-get update -qq sudo apt-get install dpkg -y sudo apt-get install -y python3-rosdep From e9cd22e5e3611333a63cf1a7db906d3c722c277c Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Mon, 25 Nov 2024 16:25:02 -0800 Subject: [PATCH 07/11] ci.yaml: Disable ebuild unit tests Temporarily disable the ebuild unit tests. Support for Gentoo needs to be updated and the tests currently fail. Signed-off-by: Rob Woolley --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e73aceaf..4c9f2970 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -45,7 +45,7 @@ jobs: run: | sudo rosdep init rosdep update - python -m 'nose' --exclude test_pull --exclude test_run --exclude test_logger_output --exclude test_docker.py --exclude test_failed_to_create + python -m 'nose' --exclude test_pull --exclude test_run --exclude test_logger_output --exclude test_failed_to_create --exclude test_generate_installers --ignore-files test_ebuild.py --ignore-files test_docker.py python -m 'flake8' superflore --import-order-style=google From f3dc27c0b32b35235d9410ad67b822e450bca6b2 Mon Sep 17 00:00:00 2001 From: Rob Woolley Date: Tue, 26 Nov 2024 18:12:53 -0800 Subject: [PATCH 08/11] ci.yaml: Increase verbosity of test run Adding --verbose to python nose will print out the test cases names. This is useful for seeing what tests are being run when viewing the logs. Signed-off-by: Rob Woolley --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4c9f2970..ba3b5d94 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -45,7 +45,7 @@ jobs: run: | sudo rosdep init rosdep update - python -m 'nose' --exclude test_pull --exclude test_run --exclude test_logger_output --exclude test_failed_to_create --exclude test_generate_installers --ignore-files test_ebuild.py --ignore-files test_docker.py + python -m 'nose' --verbose --exclude test_pull --exclude test_run --exclude test_logger_output --exclude test_failed_to_create --exclude test_generate_installers --ignore-files test_ebuild.py --ignore-files test_docker.py python -m 'flake8' superflore --import-order-style=google From 9fad4761d24afc544bce831f336f871f387d2b05 Mon Sep 17 00:00:00 2001 From: "Hunter L. Allen" Date: Fri, 6 Aug 2021 10:19:52 -0400 Subject: [PATCH 09/11] Noetic support Signed-off-by: Hunter L. Allen --- superflore/generators/ebuild/ebuild.py | 7 ++-- .../generators/ebuild/overlay_instance.py | 41 ++++++++++++++----- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/superflore/generators/ebuild/ebuild.py b/superflore/generators/ebuild/ebuild.py index e4034572..a1f956d5 100644 --- a/superflore/generators/ebuild/ebuild.py +++ b/superflore/generators/ebuild/ebuild.py @@ -138,12 +138,13 @@ def get_ebuild_text(self, distributor, license_text): # EAPI= ret = self.get_license_line(distributor, license_text) ret += self.get_eapi_line() + if self.python_3 and not self.is_ros2: # enable python 2.7 and python 3.5 ret += self.get_python_compat(['2_7', '3_5', '3_6']) - elif self.python_3: - # only use 3.5, 3.6 for ROS 2 - ret += self.get_python_compat(['3_5', '3_6']) + elif self.python_3 or (self.distro == 'noetic'): + # only use 3.5 - 3.9 for ROS 2 or noetic + ret += self.get_python_compat(['3_5', '3_6', '3_7', '3_8', '3_9']) else: # fallback to python 2.7 ret += self.get_python_compat(['2_7']) diff --git a/superflore/generators/ebuild/overlay_instance.py b/superflore/generators/ebuild/overlay_instance.py index 2ad7e055..382ba6db 100644 --- a/superflore/generators/ebuild/overlay_instance.py +++ b/superflore/generators/ebuild/overlay_instance.py @@ -60,7 +60,11 @@ def commit_changes(self, distro): self.repo.git.commit(m='{0}'.format(commit_msg)) def regenerate_manifests( - self, regen_dict, image_owner='allenh1', image_name='ros_gentoo_base' + self, + regen_dict, + image_owner='allenh1', + image_name='ros_gentoo_base', + split_limit=1000 ): info( "Pulling docker image '%s/%s:latest'..." % ( @@ -76,16 +80,31 @@ def regenerate_manifests( '/root/.gnupg' ) dock.map_directory(self.repo.repo_dir, '/tmp/ros-overlay') - for key in regen_dict.keys(): - for pkg in regen_dict[key]: - pkg_dir = '/tmp/ros-overlay/ros-{0}/{1}'.format(key, pkg) - dock.add_bash_command('cd {0}'.format(pkg_dir)) - dock.add_bash_command('repoman manifest') - try: - dock.run(show_cmd=True) - except docker.errors.ContainerError: - print(dock.log) - raise + for distro in regen_dict.keys(): + chunk_list = [] + chunk_count = 0 + pkg_list = regen_dict[distro] + while len(pkg_list) > 0: + current_chunk = list() + for x in range(split_limit): + if len(pkg_list) > 0: + current_chunk.append(pkg_list.pop()) + else: + break + chunk_list.append(current_chunk) + info("Regeneration list consists of '%d' chunks" % len(chunk_list)) + info("key_lists: '%s'" % chunk_list) + for chunk in chunk_list: + for pkg in chunk: + pkg_dir = '/tmp/ros-overlay/ros-{0}/{1}'.format(distro, pkg) + dock.add_bash_command('cd {0}'.format(pkg_dir)) + dock.add_bash_command('repoman manifest') + try: + dock.run(show_cmd=True) + dock.clear_commands() + except docker.errors.ContainerError: + print(dock.log) + raise def pull_request(self, message, overlay=None, title=''): if not title: From a8d757bd93794c30f06b051b05acdef844f5c3ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Tue, 8 Nov 2022 14:19:33 -0800 Subject: [PATCH 10/11] Remove unused argument. This `pkg` argument is shadowed by a local variable without first being used. --- superflore/generators/ebuild/gen_packages.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/superflore/generators/ebuild/gen_packages.py b/superflore/generators/ebuild/gen_packages.py index 915ec519..db844bef 100644 --- a/superflore/generators/ebuild/gen_packages.py +++ b/superflore/generators/ebuild/gen_packages.py @@ -121,7 +121,7 @@ def regenerate_pkg(overlay, pkg, distro, preserve_existing=False): def _gen_metadata_for_package( - distro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall + distro, pkg_name, repo, ros_pkg, pkg_rosinstall ): pkg_metadata_xml = metadata_xml() try: @@ -201,7 +201,7 @@ def __init__(self, distro, pkg_name, has_patches=False): self.metadata_xml =\ _gen_metadata_for_package(distro, pkg_name, - pkg, repo, ros_pkg, pkg_rosinstall) + repo, ros_pkg, pkg_rosinstall) self.ebuild =\ _gen_ebuild_for_package(distro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall) From 2f88ce75b3dc0a54f7bcb70b3737f1f889e8eccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Tue, 8 Nov 2022 14:56:04 -0800 Subject: [PATCH 11/11] Evaluate package conditions when generating ebuilds. Ebuild generation was not utilizing package conditions. Packages with multiple build types were causing hard failures during ebuild generation but it's possible that invalid ebuilds were being generated for other packages as well due to the lack of dependency conditions. --- superflore/generators/ebuild/gen_packages.py | 34 ++++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/superflore/generators/ebuild/gen_packages.py b/superflore/generators/ebuild/gen_packages.py index db844bef..d002b35b 100644 --- a/superflore/generators/ebuild/gen_packages.py +++ b/superflore/generators/ebuild/gen_packages.py @@ -120,6 +120,32 @@ def regenerate_pkg(overlay, pkg, distro, preserve_existing=False): return current, previous_version, pkg +def _package_condition_context(rosdistro_name): + distro_properties = get_distros()[rosdistro_name] + ros_version = None + if distro_properties['distribution_type'] == 'ros2': + ros_version = '2' + elif distro_properties['distribution_type'] == 'ros1': + ros_version = '1' + else: + err("Superflore does not handle the distribution type '{}'".format( + distro_properties['distribution_type'])) + raise 'Invalid distribution_type for {}'.format(rosdistro_name) + ros_python_version = None + if distro_properties['python_version'] == 3: + ros_python_version = '3' + elif distro_properties['python_version'] == 2: + ros_python_version = '2' + else: + err("Superflore does not handle the python version '{}'".format( + distro_properties['python_version'])) + raise 'Invalid python_version for {}'.format(rosdistro_name) + return { + 'ROS_DISTRO': rosdistro_name, + 'ROS_VERSION': ros_version, + 'ROS_PYTHON_VERSION': ros_python_version} + + def _gen_metadata_for_package( distro, pkg_name, repo, ros_pkg, pkg_rosinstall ): @@ -129,7 +155,8 @@ def _gen_metadata_for_package( except Exception: warn("fetch metadata for package {}".format(pkg_name)) return pkg_metadata_xml - pkg = PackageMetadata(pkg_xml) + package_condition_context = _package_condition_context(distro.name) + pkg = PackageMetadata(pkg_xml, evaluate_condition_context=package_condition_context) pkg_metadata_xml.upstream_email = pkg.upstream_email pkg_metadata_xml.upstream_name = pkg.upstream_name pkg_metadata_xml.longdescription = pkg.longdescription @@ -146,7 +173,8 @@ def _gen_ebuild_for_package( pkg_ebuild.distro = distro.name pkg_ebuild.src_uri = pkg_rosinstall[0]['tar']['uri'] pkg_names = get_package_names(distro) - pkg_dep_walker = DependencyWalker(distro) + package_condition_context = _package_condition_context(distro.name) + pkg_dep_walker = DependencyWalker(distro, evaluate_condition_context=package_condition_context) pkg_buildtool_deps = pkg_dep_walker.get_depends(pkg_name, "buildtool") pkg_build_deps = pkg_dep_walker.get_depends(pkg_name, "build") @@ -181,7 +209,7 @@ def _gen_ebuild_for_package( except Exception: warn("fetch metadata for package {}".format(pkg_name)) return pkg_ebuild - pkg = PackageMetadata(pkg_xml) + pkg = PackageMetadata(pkg_xml, evaluate_condition_context=package_condition_context) pkg_ebuild.upstream_license = pkg.upstream_license pkg_ebuild.description = pkg.description pkg_ebuild.homepage = pkg.homepage