From 0d2cf0a05397b64cf2d00aa3d06d4fa18bf655a1 Mon Sep 17 00:00:00 2001 From: gentoo90 Date: Thu, 21 Sep 2023 00:23:06 +0300 Subject: [PATCH 1/8] ebuild: close the output files or they SOMETIMES end up being empty --- superflore/generators/ebuild/gen_packages.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/superflore/generators/ebuild/gen_packages.py b/superflore/generators/ebuild/gen_packages.py index 915ec519..a9d1a1dc 100644 --- a/superflore/generators/ebuild/gen_packages.py +++ b/superflore/generators/ebuild/gen_packages.py @@ -102,18 +102,20 @@ def regenerate_pkg(overlay, pkg, distro, preserve_existing=False): ok('{0} \'{1}\'.'.format(success_msg, pkg)) try: - ebuild_file = '{0}/ros-{1}/{2}/{2}-{3}.ebuild'.format( + ebuild_file_path = '{0}/ros-{1}/{2}/{2}-{3}.ebuild'.format( overlay.repo.repo_dir, distro.name, pkg, version ) - ebuild_file = open(ebuild_file, "w") - metadata_file = '{0}/ros-{1}/{2}/metadata.xml'.format( + with open(ebuild_file_path, "w") as ebuild_file: + ebuild_file.write(ebuild_text) + + metadata_file_path = '{0}/ros-{1}/{2}/metadata.xml'.format( overlay.repo.repo_dir, distro.name, pkg ) - metadata_file = open(metadata_file, "w") - ebuild_file.write(ebuild_text) - metadata_file.write(metadata_text) + with open(metadata_file_path, "w") as metadata_file: + metadata_file.write(metadata_text) + except Exception as e: err("Failed to write ebuild/metadata to disk!") raise e From be12f3c4ea5a8e0944be37f4fe218443a4503fa3 Mon Sep 17 00:00:00 2001 From: gentoo90 Date: Thu, 21 Sep 2023 01:51:31 +0300 Subject: [PATCH 2/8] ebuild: make conditions in package.xml work Set evaluate_condition_context of DependencyWalker --- superflore/generators/ebuild/gen_packages.py | 18 ++++++++++++++---- superflore/utils.py | 10 ++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/superflore/generators/ebuild/gen_packages.py b/superflore/generators/ebuild/gen_packages.py index a9d1a1dc..37060c1b 100644 --- a/superflore/generators/ebuild/gen_packages.py +++ b/superflore/generators/ebuild/gen_packages.py @@ -27,6 +27,8 @@ from superflore.utils import err from superflore.utils import get_distros from superflore.utils import get_pkg_version +from superflore.utils import get_ros_version +from superflore.utils import get_ros_python_version from superflore.utils import make_dir from superflore.utils import ok from superflore.utils import retry_on_exception @@ -50,7 +52,7 @@ def regenerate_pkg(overlay, pkg, distro, preserve_existing=False): ebuild_name = overlay.repo.repo_dir + ebuild_name patch_path = '/ros-{}/{}/files'.format(distro.name, pkg) patch_path = overlay.repo.repo_dir + patch_path - is_ros2 = get_distros()[distro.name]['distribution_type'] == 'ros2' + is_ros2 = get_ros_version(distro.name) == 2 has_patches = os.path.exists(patch_path) pkg_names = get_package_names(distro)[0] patches = None @@ -131,7 +133,7 @@ def _gen_metadata_for_package( except Exception: warn("fetch metadata for package {}".format(pkg_name)) return pkg_metadata_xml - pkg = PackageMetadata(pkg_xml) + pkg = PackageMetadata(pkg_xml, _get_evaluation_context(distro)) pkg_metadata_xml.upstream_email = pkg.upstream_email pkg_metadata_xml.upstream_name = pkg.upstream_name pkg_metadata_xml.longdescription = pkg.longdescription @@ -148,7 +150,7 @@ def _gen_ebuild_for_package( pkg_ebuild.distro = distro.name pkg_ebuild.src_uri = pkg_rosinstall[0]['tar']['uri'] pkg_names = get_package_names(distro) - pkg_dep_walker = DependencyWalker(distro) + pkg_dep_walker = DependencyWalker(distro, evaluate_condition_context=_get_evaluation_context(distro)) pkg_buildtool_deps = pkg_dep_walker.get_depends(pkg_name, "buildtool") pkg_build_deps = pkg_dep_walker.get_depends(pkg_name, "build") @@ -183,7 +185,7 @@ def _gen_ebuild_for_package( except Exception: warn("fetch metadata for package {}".format(pkg_name)) return pkg_ebuild - pkg = PackageMetadata(pkg_xml) + pkg = PackageMetadata(pkg_xml, _get_evaluation_context(distro)) pkg_ebuild.upstream_license = pkg.upstream_license pkg_ebuild.description = pkg.description pkg_ebuild.homepage = pkg.homepage @@ -191,6 +193,14 @@ def _gen_ebuild_for_package( return pkg_ebuild +def _get_evaluation_context(distro): + return { + "ROS_DISTRO": distro.name, + "ROS_VERSION": str(get_ros_version(distro.name)), + "ROS_PYTHON_VERSION": str(get_ros_python_version(distro.name)) + } + + class gentoo_ebuild(object): def __init__(self, distro, pkg_name, has_patches=False): pkg = distro.release_packages[pkg_name] diff --git a/superflore/utils.py b/superflore/utils.py index 8fb5cc71..410f16a8 100644 --- a/superflore/utils.py +++ b/superflore/utils.py @@ -715,6 +715,16 @@ def get_distros_by_status(status='active'): if t[1].get('distribution_status') == status] +def get_ros_version(distro_name): + distros = get_distros() + return 2 if distro_name not in distros \ + else int(distros[distro_name]['distribution_type'][len('ros'):]) + + +def get_ros_python_version(distro_name): + return 2 if distro_name in ['melodic'] else 3 + + def gen_delta_msg(total_changes, markup='*'): """Return string of changes for the PR message.""" delta = '' From de64be6b4c2c025779723bfadd7e11469f527f4d Mon Sep 17 00:00:00 2001 From: gentoo90 Date: Thu, 21 Sep 2023 01:54:14 +0300 Subject: [PATCH 3/8] ebuild: EAPI6 is dead, long live EAPI8 --- superflore/generators/ebuild/ebuild.py | 3 ++- tests/ebuild/simple_expected.ebuild | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/superflore/generators/ebuild/ebuild.py b/superflore/generators/ebuild/ebuild.py index e4034572..c0d12064 100644 --- a/superflore/generators/ebuild/ebuild.py +++ b/superflore/generators/ebuild/ebuild.py @@ -49,8 +49,9 @@ class Ebuild(object): Basic definition of an ebuild. This is where any necessary variables will be filled. """ + def __init__(self): - self.eapi = str(6) + self.eapi = str(8) self.description = "" self.homepage = "https://wiki.ros.org" self.src_uri = None diff --git a/tests/ebuild/simple_expected.ebuild b/tests/ebuild/simple_expected.ebuild index 714c101e..1237a4bb 100644 --- a/tests/ebuild/simple_expected.ebuild +++ b/tests/ebuild/simple_expected.ebuild @@ -1,7 +1,7 @@ # Copyright 2017 Open Source Robotics Foundation # Distributed under the terms of the BSD license -EAPI=6 +EAPI=8 PYTHON_COMPAT=( python{2_7,3_5,3_6} ) inherit ros-cmake From 6dd99a3e032b915db11c0b8c560e973f28efa8f5 Mon Sep 17 00:00:00 2001 From: gentoo90 Date: Thu, 21 Sep 2023 01:55:11 +0300 Subject: [PATCH 4/8] ebuild: add buildtool_export and build_export to the dependencies --- superflore/generators/ebuild/gen_packages.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/superflore/generators/ebuild/gen_packages.py b/superflore/generators/ebuild/gen_packages.py index 37060c1b..3a5cefb3 100644 --- a/superflore/generators/ebuild/gen_packages.py +++ b/superflore/generators/ebuild/gen_packages.py @@ -152,8 +152,14 @@ def _gen_ebuild_for_package( pkg_names = get_package_names(distro) pkg_dep_walker = DependencyWalker(distro, evaluate_condition_context=_get_evaluation_context(distro)) - pkg_buildtool_deps = pkg_dep_walker.get_depends(pkg_name, "buildtool") - pkg_build_deps = pkg_dep_walker.get_depends(pkg_name, "build") + pkg_buildtool_deps = list(set([ + *pkg_dep_walker.get_depends(pkg_name, "buildtool"), + *pkg_dep_walker.get_depends(pkg_name, "buildtool_export"), + ])) + pkg_build_deps = list(set([ + *pkg_dep_walker.get_depends(pkg_name, "build"), + *pkg_dep_walker.get_depends(pkg_name, "build_export"), + ])) pkg_run_deps = pkg_dep_walker.get_depends(pkg_name, "run") pkg_test_deps = pkg_dep_walker.get_depends(pkg_name, "test") From eec9701c4852eb4eb5b9fd50354706a549620365 Mon Sep 17 00:00:00 2001 From: gentoo90 Date: Thu, 21 Sep 2023 01:56:39 +0300 Subject: [PATCH 5/8] ebuild: patches are now applied via PATCHES variable --- superflore/generators/ebuild/ebuild.py | 13 ++++--------- superflore/generators/ebuild/gen_packages.py | 2 +- tests/test_ebuild.py | 10 ++-------- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/superflore/generators/ebuild/ebuild.py b/superflore/generators/ebuild/ebuild.py index c0d12064..18ef7e58 100644 --- a/superflore/generators/ebuild/ebuild.py +++ b/superflore/generators/ebuild/ebuild.py @@ -232,15 +232,10 @@ def get_ebuild_text(self, distributor, license_text): # Patch source if needed. if self.has_patches: - # TODO(allenh1): explicitly list patches - ret += "\nsrc_prepare() {\n" - ret += " cd ${P}\n" - ret += " EPATCH_SOURCE=\"${FILESDIR}\"" - ret += " EPATCH_SUFFIX=\"patch\" \\\n" - ret += " EPATCH_FORCE=\"yes\" epatch\n" - if self.build_type in ['catkin', 'cmake']: - ret += " ros-cmake_src_prepare\n" - ret += "}\n" + ret += "\nPATCHES=(\n" + for patch in self.patches: + ret += " \"${{FILESDIR}}/{}\"\n".format(patch) + ret += ")\n" # source configuration if self.name == 'opencv3': diff --git a/superflore/generators/ebuild/gen_packages.py b/superflore/generators/ebuild/gen_packages.py index 3a5cefb3..18b99a1b 100644 --- a/superflore/generators/ebuild/gen_packages.py +++ b/superflore/generators/ebuild/gen_packages.py @@ -58,7 +58,7 @@ def regenerate_pkg(overlay, pkg, distro, preserve_existing=False): patches = None if os.path.exists(patch_path): patches = [ - f for f in glob.glob('%s/*.patch' % patch_path) + f for f in glob.glob('*.patch', root_dir=patch_path) ] if pkg not in pkg_names: raise RuntimeError("Unknown package '%s'" % (pkg)) diff --git a/tests/test_ebuild.py b/tests/test_ebuild.py index 36c732cd..9112ba77 100644 --- a/tests/test_ebuild.py +++ b/tests/test_ebuild.py @@ -160,19 +160,13 @@ def test_has_patches(self): ebuild = self.get_ebuild() ebuild.has_patches = True; got_text = ebuild.get_ebuild_text('Open Source Robotics Foundation', 'BSD') - self.assertTrue('EPATCH_SOURCE="${FILESDIR}"' in got_text) - self.assertTrue('EPATCH_SUFFIX="patch"' in got_text) - self.assertTrue('EPATCH_FORCE="yes"' in got_text) - self.assertTrue('epatch' in got_text) + self.assertTrue('PATCHES=(' in got_text) def test_lacks_patches(self): """Test Non-Patched Code Generation""" ebuild = self.get_ebuild() got_text = ebuild.get_ebuild_text('Open Source Robotics Foundation', 'BSD') - self.assertFalse('EPATCH_SOURCE="${FILESDIR}"' in got_text) - self.assertFalse('EPATCH_SUFFIX="patch"' in got_text) - self.assertFalse('EPATCH_FORCE="yes"' in got_text) - self.assertFalse('epatch' in got_text) + self.assertFalse('PATCHES=(' in got_text) def test_opencv3_filter_flags(self): """Test Filter Flags for OpenCV3""" From 8d75ef2db1292173ffb493a59435d5f43be803ad Mon Sep 17 00:00:00 2001 From: gentoo90 Date: Fri, 22 Sep 2023 21:49:34 +0300 Subject: [PATCH 6/8] ebuild: switch to newer versions of Python --- superflore/generators/ebuild/ebuild.py | 4 ++-- tests/ebuild/simple_expected.ebuild | 2 +- tests/test_ebuild.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/superflore/generators/ebuild/ebuild.py b/superflore/generators/ebuild/ebuild.py index 18ef7e58..06abf1d0 100644 --- a/superflore/generators/ebuild/ebuild.py +++ b/superflore/generators/ebuild/ebuild.py @@ -141,10 +141,10 @@ def get_ebuild_text(self, distributor, license_text): ret += self.get_eapi_line() if self.python_3 and not self.is_ros2: # enable python 2.7 and python 3.5 - ret += self.get_python_compat(['2_7', '3_5', '3_6']) + ret += self.get_python_compat(['2_7', '3_10', '3_11']) elif self.python_3: # only use 3.5, 3.6 for ROS 2 - ret += self.get_python_compat(['3_5', '3_6']) + ret += self.get_python_compat(['3_10', '3_11']) else: # fallback to python 2.7 ret += self.get_python_compat(['2_7']) diff --git a/tests/ebuild/simple_expected.ebuild b/tests/ebuild/simple_expected.ebuild index 1237a4bb..09c38c7a 100644 --- a/tests/ebuild/simple_expected.ebuild +++ b/tests/ebuild/simple_expected.ebuild @@ -2,7 +2,7 @@ # Distributed under the terms of the BSD license EAPI=8 -PYTHON_COMPAT=( python{2_7,3_5,3_6} ) +PYTHON_COMPAT=( python{2_7,3_10,3_11} ) inherit ros-cmake diff --git a/tests/test_ebuild.py b/tests/test_ebuild.py index 9112ba77..4802cc71 100644 --- a/tests/test_ebuild.py +++ b/tests/test_ebuild.py @@ -153,7 +153,7 @@ def test_default_python2_python3(self): """Test That Python2/3 Is the Default""" ebuild = self.get_ebuild() got_text = ebuild.get_ebuild_text('Open Source Robotics Foundation', 'BSD') - self.assertTrue('PYTHON_COMPAT=( python{2_7,3_5,3_6} )' in got_text) + self.assertTrue('PYTHON_COMPAT=( python{2_7,3_10,3_11} )' in got_text) def test_has_patches(self): """Test Patch Code Generation""" From 356e9173b3b083ce96bccd55af3d47419ddb7264 Mon Sep 17 00:00:00 2001 From: gentoo90 Date: Sun, 24 Sep 2023 02:54:41 +0300 Subject: [PATCH 7/8] ebuild: set default HOMEPAGE to the packages page in ROS index --- superflore/PackageMetadata.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/superflore/PackageMetadata.py b/superflore/PackageMetadata.py index cb0f5b5d..faab83af 100644 --- a/superflore/PackageMetadata.py +++ b/superflore/PackageMetadata.py @@ -33,6 +33,11 @@ def __init__(self, pkg_xml, evaluate_condition_context=None): self.homepage = [ url.url for url in pkg.urls ][0] + else: + self.homepage = "https://index.ros.org/p/{}/".format(pkg.name) + if evaluate_condition_context is not None: + distro_name = evaluate_condition_context['ROS_DISTRO'] + self.homepage += '#{}'.format(distro_name) self.longdescription = pkg.description self.upstream_email = [ author.email for author in pkg.maintainers From 47b7672cc803402541bdc340bedb81e37cd012e3 Mon Sep 17 00:00:00 2001 From: gentoo90 Date: Fri, 19 Jan 2024 22:42:10 +0200 Subject: [PATCH 8/8] Update OS and Python versions in the CI config --- .github/workflows/ci.yaml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fba3e7c9..cfea786c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,14 +6,14 @@ on: pull_request: jobs: - build: - strategy: - matrix: - os: [ubuntu-18.04, ubuntu-20.04] - python: [3.6, 3.7, 3.8, 3.9] - name: superflore tests - runs-on: ${{matrix.os}} - steps: + build: + strategy: + matrix: + os: [ubuntu-20.04, ubuntu-22.04] + python: [3.10, 3.11] + name: superflore tests + runs-on: ${{matrix.os}} + steps: - uses: actions/checkout@v2 - name: Set up Python ${{matrix.python}} uses: actions/setup-python@v1 @@ -34,6 +34,3 @@ jobs: rosdep update python -m 'nose' --exclude test_pull --exclude test_run --exclude test_logger_output python -m 'flake8' superflore --import-order-style=google - - -