Update ebuild generator to support the latest systems

.github/workflows/ci.yaml

@@ -6,14 +6,14 @@ on:
   pull_request:
 
 jobs:
-    build:
-      strategy:
-        matrix:
-          os: [ubuntu-18.04, ubuntu-20.04]
-          python: [3.6, 3.7, 3.8, 3.9]
-      name: superflore tests
-      runs-on: ${{matrix.os}}
-      steps:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-20.04, ubuntu-22.04]
+        python: [3.10, 3.11]
+    name: superflore tests
+    runs-on: ${{matrix.os}}
+    steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{matrix.python}}
         uses: actions/setup-python@v1
@@ -34,6 +34,3 @@ jobs:
           rosdep update
           python -m 'nose' --exclude test_pull --exclude test_run --exclude test_logger_output
           python -m 'flake8' superflore --import-order-style=google
-
-
-

superflore/PackageMetadata.py

@@ -33,6 +33,11 @@ def __init__(self, pkg_xml, evaluate_condition_context=None):
             self.homepage = [
                 url.url for url in pkg.urls
             ][0]
+        else:
+            self.homepage = "https://index.ros.org/p/{}/".format(pkg.name)
+            if evaluate_condition_context is not None:
+                distro_name = evaluate_condition_context['ROS_DISTRO']
+                self.homepage += '#{}'.format(distro_name)
         self.longdescription = pkg.description
         self.upstream_email = [
             author.email for author in pkg.maintainers

superflore/generators/ebuild/ebuild.py

@@ -49,8 +49,9 @@ class Ebuild(object):
     Basic definition of an ebuild.
     This is where any necessary variables will be filled.
     """
+
     def __init__(self):
-        self.eapi = str(6)
+        self.eapi = str(8)
         self.description = ""
         self.homepage = "https://wiki.ros.org"
         self.src_uri = None
@@ -140,10 +141,10 @@ def get_ebuild_text(self, distributor, license_text):
         ret += self.get_eapi_line()
         if self.python_3 and not self.is_ros2:
             # enable python 2.7 and python 3.5
-            ret += self.get_python_compat(['2_7', '3_5', '3_6'])
+            ret += self.get_python_compat(['2_7', '3_10', '3_11'])
         elif self.python_3:
             # only use 3.5, 3.6 for ROS 2
-            ret += self.get_python_compat(['3_5', '3_6'])
+            ret += self.get_python_compat(['3_10', '3_11'])
         else:
             # fallback to python 2.7
             ret += self.get_python_compat(['2_7'])
@@ -231,15 +232,10 @@ def get_ebuild_text(self, distributor, license_text):
 
         # Patch source if needed.
         if self.has_patches:
-            # TODO(allenh1): explicitly list patches
-            ret += "\nsrc_prepare() {\n"
-            ret += "    cd ${P}\n"
-            ret += "    EPATCH_SOURCE=\"${FILESDIR}\""
-            ret += " EPATCH_SUFFIX=\"patch\" \\\n"
-            ret += "    EPATCH_FORCE=\"yes\" epatch\n"
-            if self.build_type in ['catkin', 'cmake']:
-                ret += "    ros-cmake_src_prepare\n"
-            ret += "}\n"
+            ret += "\nPATCHES=(\n"
+            for patch in self.patches:
+                ret += "    \"${{FILESDIR}}/{}\"\n".format(patch)
+            ret += ")\n"
 
         # source configuration
         if self.name == 'opencv3':

superflore/generators/ebuild/gen_packages.py

@@ -27,6 +27,8 @@
 from superflore.utils import err
 from superflore.utils import get_distros
 from superflore.utils import get_pkg_version
+from superflore.utils import get_ros_version
+from superflore.utils import get_ros_python_version
 from superflore.utils import make_dir
 from superflore.utils import ok
 from superflore.utils import retry_on_exception
@@ -50,13 +52,13 @@ def regenerate_pkg(overlay, pkg, distro, preserve_existing=False):
     ebuild_name = overlay.repo.repo_dir + ebuild_name
     patch_path = '/ros-{}/{}/files'.format(distro.name, pkg)
     patch_path = overlay.repo.repo_dir + patch_path
-    is_ros2 = get_distros()[distro.name]['distribution_type'] == 'ros2'
+    is_ros2 = get_ros_version(distro.name) == 2
     has_patches = os.path.exists(patch_path)
     pkg_names = get_package_names(distro)[0]
     patches = None
     if os.path.exists(patch_path):
         patches = [
-            f for f in glob.glob('%s/*.patch' % patch_path)
+            f for f in glob.glob('*.patch', root_dir=patch_path)
         ]
     if pkg not in pkg_names:
         raise RuntimeError("Unknown package '%s'" % (pkg))
@@ -102,18 +104,20 @@ def regenerate_pkg(overlay, pkg, distro, preserve_existing=False):
     ok('{0} \'{1}\'.'.format(success_msg, pkg))
 
     try:
-        ebuild_file = '{0}/ros-{1}/{2}/{2}-{3}.ebuild'.format(
+        ebuild_file_path = '{0}/ros-{1}/{2}/{2}-{3}.ebuild'.format(
             overlay.repo.repo_dir,
             distro.name, pkg, version
         )
-        ebuild_file = open(ebuild_file, "w")
-        metadata_file = '{0}/ros-{1}/{2}/metadata.xml'.format(
+        with open(ebuild_file_path, "w") as ebuild_file:
+            ebuild_file.write(ebuild_text)
+
+        metadata_file_path = '{0}/ros-{1}/{2}/metadata.xml'.format(
             overlay.repo.repo_dir,
             distro.name, pkg
         )
-        metadata_file = open(metadata_file, "w")
-        ebuild_file.write(ebuild_text)
-        metadata_file.write(metadata_text)
+        with open(metadata_file_path, "w") as metadata_file:
+            metadata_file.write(metadata_text)
+
     except Exception as e:
         err("Failed to write ebuild/metadata to disk!")
         raise e
@@ -129,7 +133,7 @@ def _gen_metadata_for_package(
     except Exception:
         warn("fetch metadata for package {}".format(pkg_name))
         return pkg_metadata_xml
-    pkg = PackageMetadata(pkg_xml)
+    pkg = PackageMetadata(pkg_xml, _get_evaluation_context(distro))
     pkg_metadata_xml.upstream_email = pkg.upstream_email
     pkg_metadata_xml.upstream_name = pkg.upstream_name
     pkg_metadata_xml.longdescription = pkg.longdescription
@@ -146,10 +150,16 @@ def _gen_ebuild_for_package(
     pkg_ebuild.distro = distro.name
     pkg_ebuild.src_uri = pkg_rosinstall[0]['tar']['uri']
     pkg_names = get_package_names(distro)
-    pkg_dep_walker = DependencyWalker(distro)
-
-    pkg_buildtool_deps = pkg_dep_walker.get_depends(pkg_name, "buildtool")
-    pkg_build_deps = pkg_dep_walker.get_depends(pkg_name, "build")
+    pkg_dep_walker = DependencyWalker(distro, evaluate_condition_context=_get_evaluation_context(distro))
+
+    pkg_buildtool_deps = list(set([
+        *pkg_dep_walker.get_depends(pkg_name, "buildtool"),
+        *pkg_dep_walker.get_depends(pkg_name, "buildtool_export"),
+    ]))
+    pkg_build_deps = list(set([
+        *pkg_dep_walker.get_depends(pkg_name, "build"),
+        *pkg_dep_walker.get_depends(pkg_name, "build_export"),
+    ]))
     pkg_run_deps = pkg_dep_walker.get_depends(pkg_name, "run")
     pkg_test_deps = pkg_dep_walker.get_depends(pkg_name, "test")
 
@@ -181,14 +191,22 @@ def _gen_ebuild_for_package(
     except Exception:
         warn("fetch metadata for package {}".format(pkg_name))
         return pkg_ebuild
-    pkg = PackageMetadata(pkg_xml)
+    pkg = PackageMetadata(pkg_xml, _get_evaluation_context(distro))
     pkg_ebuild.upstream_license = pkg.upstream_license
     pkg_ebuild.description = pkg.description
     pkg_ebuild.homepage = pkg.homepage
     pkg_ebuild.build_type = pkg.build_type
     return pkg_ebuild
 
 
+def _get_evaluation_context(distro):
+    return {
+        "ROS_DISTRO": distro.name,
+        "ROS_VERSION": str(get_ros_version(distro.name)),
+        "ROS_PYTHON_VERSION": str(get_ros_python_version(distro.name))
+    }
+
+
 class gentoo_ebuild(object):
     def __init__(self, distro, pkg_name, has_patches=False):
         pkg = distro.release_packages[pkg_name]

superflore/utils.py

@@ -715,6 +715,16 @@ def get_distros_by_status(status='active'):
             if t[1].get('distribution_status') == status]
 
 
+def get_ros_version(distro_name):
+    distros = get_distros()
+    return 2 if distro_name not in distros \
+        else int(distros[distro_name]['distribution_type'][len('ros'):])
+
+
+def get_ros_python_version(distro_name):
+    return 2 if distro_name in ['melodic'] else 3
+
+
 def gen_delta_msg(total_changes, markup='*'):
     """Return string of changes for the PR message."""
     delta = ''

tests/ebuild/simple_expected.ebuild

@@ -1,8 +1,8 @@
 # Copyright 2017 Open Source Robotics Foundation
 # Distributed under the terms of the BSD license
 
-EAPI=6
-PYTHON_COMPAT=( python{2_7,3_5,3_6} )
+EAPI=8
+PYTHON_COMPAT=( python{2_7,3_10,3_11} )
 
 inherit ros-cmake
 

tests/test_ebuild.py

@@ -153,26 +153,20 @@ def test_default_python2_python3(self):
         """Test That Python2/3 Is the Default"""
         ebuild = self.get_ebuild()
         got_text = ebuild.get_ebuild_text('Open Source Robotics Foundation', 'BSD')
-        self.assertTrue('PYTHON_COMPAT=( python{2_7,3_5,3_6} )' in got_text)
+        self.assertTrue('PYTHON_COMPAT=( python{2_7,3_10,3_11} )' in got_text)
 
     def test_has_patches(self):
         """Test Patch Code Generation"""
         ebuild = self.get_ebuild()
         ebuild.has_patches = True;
         got_text = ebuild.get_ebuild_text('Open Source Robotics Foundation', 'BSD')
-        self.assertTrue('EPATCH_SOURCE="${FILESDIR}"' in got_text)
-        self.assertTrue('EPATCH_SUFFIX="patch"' in got_text)
-        self.assertTrue('EPATCH_FORCE="yes"' in got_text)
-        self.assertTrue('epatch' in got_text)
+        self.assertTrue('PATCHES=(' in got_text)
 
     def test_lacks_patches(self):
         """Test Non-Patched Code Generation"""
         ebuild = self.get_ebuild()
         got_text = ebuild.get_ebuild_text('Open Source Robotics Foundation', 'BSD')
-        self.assertFalse('EPATCH_SOURCE="${FILESDIR}"' in got_text)
-        self.assertFalse('EPATCH_SUFFIX="patch"' in got_text)
-        self.assertFalse('EPATCH_FORCE="yes"' in got_text)
-        self.assertFalse('epatch' in got_text)
+        self.assertFalse('PATCHES=(' in got_text)
 
     def test_opencv3_filter_flags(self):
         """Test Filter Flags for OpenCV3"""