From 83b8a7e511887500346f601394fca0661428ad04 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Fri, 6 Dec 2024 12:57:39 -0600 Subject: [PATCH] Add a GitLab manifest provider (#163) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Steven! Ragnarök --- src/rosdistro/distribution.py | 5 +- src/rosdistro/manifest_provider/gitlab.py | 158 ++++++++++++++++++++++ src/rosdistro/release.py | 3 +- test/test_manifest_providers.py | 32 +++++ 4 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 src/rosdistro/manifest_provider/gitlab.py diff --git a/src/rosdistro/distribution.py b/src/rosdistro/distribution.py index 84055c6a6..742cc638b 100644 --- a/src/rosdistro/distribution.py +++ b/src/rosdistro/distribution.py @@ -34,14 +34,15 @@ from .manifest_provider.bitbucket import bitbucket_manifest_provider from .manifest_provider.git import git_manifest_provider, git_source_manifest_provider from .manifest_provider.github import github_manifest_provider, github_source_manifest_provider +from .manifest_provider.gitlab import gitlab_manifest_provider, gitlab_source_manifest_provider from .manifest_provider.tar import tar_manifest_provider, tar_source_manifest_provider from .package import Package class Distribution(object): - default_manifest_providers = [github_manifest_provider, bitbucket_manifest_provider, git_manifest_provider, tar_manifest_provider] - default_source_manifest_providers = [github_source_manifest_provider, git_source_manifest_provider, tar_source_manifest_provider] + default_manifest_providers = [github_manifest_provider, gitlab_manifest_provider, bitbucket_manifest_provider, git_manifest_provider, tar_manifest_provider] + default_source_manifest_providers = [github_source_manifest_provider, gitlab_source_manifest_provider, git_source_manifest_provider, tar_source_manifest_provider] def __init__(self, distribution_file, manifest_providers=None, source_manifest_providers=None): self._distribution_file = distribution_file diff --git a/src/rosdistro/manifest_provider/gitlab.py b/src/rosdistro/manifest_provider/gitlab.py new file mode 100644 index 000000000..884504537 --- /dev/null +++ b/src/rosdistro/manifest_provider/gitlab.py @@ -0,0 +1,158 @@ +# Software License Agreement (BSD License) +# +# Copyright (c) 2021, Open Source Robotics Foundation, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided +# with the distribution. +# * Neither the name of Open Source Robotics Foundation, Inc. nor +# the names of its contributors may be used to endorse or promote +# products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +import json +import os +import re +from urllib.request import urlopen, Request +from urllib.error import URLError +from urllib.parse import quote as urlquote +from urllib.parse import urlencode + +from catkin_pkg.package import parse_package_string + +from rosdistro.source_repository_cache import SourceRepositoryCache +from rosdistro import logger + +GITLAB_PRIVATE_TOKEN = os.getenv('GITLAB_PRIVATE_TOKEN', None) +ROSDISTRO_GITLAB_SERVER = os.getenv('ROSDISTRO_GITLAB_SERVER', None) + +def _gitlab_urlopen(url): + req = Request(url) + if GITLAB_PRIVATE_TOKEN: + req.add_header('Private-Token', GITLAB_PRIVATE_TOKEN) + logger.warn('Performing GitLab API query "%s"' % (url,)) + return urlopen(req) + + +def _gitlab_api_query(server, path, resource, attrs): + url = 'https://%s/api/v4/projects/%s/%s' % (server, urlquote(path, safe=''), resource) + if attrs: + url += '?' + urlencode(attrs) + return _gitlab_urlopen(url) + + +def _gitlab_paged_api_query(server, path, resource, attrs): + _attrs = { + 'per_page': 50, + **attrs, + 'pagination': 'keyset', + 'page': '1', + } + + url = 'https://%s/api/v4/projects/%s/%s' % (server, urlquote(path, safe=''), resource) + if _attrs: + url += '?' + urlencode(_attrs) + + while True: + with _gitlab_urlopen(url) as res: + for result in json.loads(res.read().decode('utf-8')): + yield result + + # Get the URL to the next page + links = res.getheader('Link') + if not links: + break + match = re.match(r'.*<([^>]*)>; rel="next"', links) + if not match: + break + url = match.group(1) + + +def gitlab_manifest_provider(_dist_name, repo, pkg_name): + assert repo.version + server, path = repo.get_url_parts() + if not server.endswith('gitlab.com') and server != ROSDISTRO_GITLAB_SERVER: + logger.debug('Skip non-gitlab url "%s"' % repo.url) + raise RuntimeError('can not handle non gitlab urls') + + resource = 'repository/files/package.xml/raw' + attrs = { + 'ref': repo.get_release_tag(pkg_name), + } + try: + with _gitlab_api_query(server, path, resource, attrs) as res: + return res.read().decode('utf-8') + except URLError as e: + logger.debug('- failed (%s), trying "%s"' % (e, e.filename)) + raise + + +def gitlab_source_manifest_provider(repo): + assert repo.version + server, path = repo.get_url_parts() + if not server.endswith('gitlab.com') and server != ROSDISTRO_GITLAB_SERVER: + logger.debug('Skip non-gitlab url "%s"' % repo.url) + raise RuntimeError('can not handle non gitlab urls') + + # Resolve the version ref to a sha since we need to make multiple queries + attrs = { + 'per_page': 1, + 'ref_name': repo.version, + } + sha = next(_gitlab_paged_api_query(server, path, 'repository/commits', attrs))['id'] + + # Look for package.xml files in the tree + attrs = { + 'recursive': 'true', + 'ref': sha, + } + package_xml_paths = set() + for obj in _gitlab_paged_api_query(server, path, 'repository/tree', attrs): + if obj['path'].split('/')[-1] == 'package.xml': + package_xml_paths.add(os.path.dirname(obj['path'])) + + # Filter out ones that are inside other packages (eg, part of tests) + def package_xml_in_parent(path): + if path == '': + return True + parent = path + while True: + parent = os.path.dirname(parent) + if parent in package_xml_paths: + return False + if parent == '': + return True + package_xml_paths = list(filter(package_xml_in_parent, package_xml_paths)) + + cache = SourceRepositoryCache.from_ref(sha) + for package_xml_path in package_xml_paths: + resource_path = urlquote( + package_xml_path + '/package.xml' if package_xml_path else 'package.xml', safe='') + resource = 'repository/files/' + resource_path + '/raw' + with _gitlab_api_query(server, path, resource, {'ref': sha}) as res: + package_xml = res.read().decode('utf-8') + name = parse_package_string(package_xml).name + cache.add(name, package_xml_path, package_xml) + + return cache diff --git a/src/rosdistro/release.py b/src/rosdistro/release.py index 1ceb251d8..687b06919 100644 --- a/src/rosdistro/release.py +++ b/src/rosdistro/release.py @@ -34,11 +34,12 @@ from .manifest_provider.bitbucket import bitbucket_manifest_provider from .manifest_provider.git import git_manifest_provider from .manifest_provider.github import github_manifest_provider +from .manifest_provider.gitlab import gitlab_manifest_provider class Release(object): - default_manifest_providers = [github_manifest_provider, bitbucket_manifest_provider, git_manifest_provider] + default_manifest_providers = [github_manifest_provider, gitlab_manifest_provider, bitbucket_manifest_provider, git_manifest_provider] def __init__(self, rel_file, manifest_providers=None): self._rel_file = rel_file diff --git a/test/test_manifest_providers.py b/test/test_manifest_providers.py index c6a88706b..d0d96482e 100644 --- a/test/test_manifest_providers.py +++ b/test/test_manifest_providers.py @@ -9,6 +9,7 @@ from rosdistro.manifest_provider.bitbucket import bitbucket_manifest_provider from rosdistro.manifest_provider.cache import CachedManifestProvider, sanitize_xml from rosdistro.manifest_provider.git import git_manifest_provider, git_source_manifest_provider +from rosdistro.manifest_provider.gitlab import gitlab_manifest_provider, gitlab_source_manifest_provider from rosdistro.release_repository_specification import ReleaseRepositorySpecification from rosdistro.source_repository_specification import SourceRepositorySpecification @@ -17,6 +18,10 @@ def test_bitbucket(): assert '' in bitbucket_manifest_provider('indigo', _rospeex_release_repo(), 'rospeex_msgs') +def test_gitlab(): + assert '' in gitlab_manifest_provider('foxy', _tracetools_analysis_release_repo(), 'tracetools_analysis') + + def test_cached(): class FakeDistributionCache(object): def __init__(self): @@ -97,6 +102,17 @@ def test_github_source(): assert '0.5.11' in package_xml +def test_gitlab_source(): + repo_cache = gitlab_source_manifest_provider(_tracetools_analysis_source_repo()) + + # This hash corresponds to the 1.0.3 tag. + assert repo_cache.ref() == 'cd30853005ef3a591cb8594b4aa49f9ef400d30f' + + package_path, package_xml = repo_cache['ros2trace_analysis'] + assert 'ros2trace_analysis' == package_path + assert '1.0.3' in package_xml + + def test_git_source_multi(): repo_cache = git_source_manifest_provider(_ros_source_repo()) assert repo_cache.ref() @@ -170,3 +186,19 @@ def _rospeex_release_repo(): 'url': 'https://bitbucket.org/rospeex/rospeex-release.git', 'version': '2.14.7-0' }) + + +def _tracetools_analysis_release_repo(): + return ReleaseRepositorySpecification('tracetools_analysis', { + 'packages': ['ros2trace_analysis', 'tracetools_analysis'], + 'tags': {'release': 'release/foxy/{package}/{version}'}, + 'url': 'https://gitlab.com/ros-tracing/tracetools_analysis-release.git', + 'version': '1.0.3-1' + }) + + +def _tracetools_analysis_source_repo(): + return SourceRepositorySpecification('tracetools_analysis', { + 'url': 'https://gitlab.com/ros-tracing/tracetools_analysis.git', + 'version': '1.0.3' + })