Add a GitLab manifest provider (#163)

Co-authored-by: Steven! Ragnarök <[email protected]>

src/rosdistro/distribution.py

@@ -34,14 +34,15 @@
 from .manifest_provider.bitbucket import bitbucket_manifest_provider
 from .manifest_provider.git import git_manifest_provider, git_source_manifest_provider
 from .manifest_provider.github import github_manifest_provider, github_source_manifest_provider
+from .manifest_provider.gitlab import gitlab_manifest_provider, gitlab_source_manifest_provider
 from .manifest_provider.tar import tar_manifest_provider, tar_source_manifest_provider
 from .package import Package
 
 
 class Distribution(object):
 
-    default_manifest_providers = [github_manifest_provider, bitbucket_manifest_provider, git_manifest_provider, tar_manifest_provider]
-    default_source_manifest_providers = [github_source_manifest_provider, git_source_manifest_provider, tar_source_manifest_provider]
+    default_manifest_providers = [github_manifest_provider, gitlab_manifest_provider, bitbucket_manifest_provider, git_manifest_provider, tar_manifest_provider]
+    default_source_manifest_providers = [github_source_manifest_provider, gitlab_source_manifest_provider, git_source_manifest_provider, tar_source_manifest_provider]
 
     def __init__(self, distribution_file, manifest_providers=None, source_manifest_providers=None):
         self._distribution_file = distribution_file

src/rosdistro/manifest_provider/gitlab.py

@@ -0,0 +1,158 @@
+# Software License Agreement (BSD License)
+#
+# Copyright (c) 2021, Open Source Robotics Foundation, Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+#
+#  * Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+#  * Redistributions in binary form must reproduce the above
+#    copyright notice, this list of conditions and the following
+#    disclaimer in the documentation and/or other materials provided
+#    with the distribution.
+#  * Neither the name of Open Source Robotics Foundation, Inc. nor
+#    the names of its contributors may be used to endorse or promote
+#    products derived from this software without specific prior
+#    written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+import json
+import os
+import re
+from urllib.request import urlopen, Request
+from urllib.error import URLError
+from urllib.parse import quote as urlquote
+from urllib.parse import urlencode
+
+from catkin_pkg.package import parse_package_string
+
+from rosdistro.source_repository_cache import SourceRepositoryCache
+from rosdistro import logger
+
+GITLAB_PRIVATE_TOKEN = os.getenv('GITLAB_PRIVATE_TOKEN', None)
+ROSDISTRO_GITLAB_SERVER = os.getenv('ROSDISTRO_GITLAB_SERVER', None)
+
+def _gitlab_urlopen(url):
+    req = Request(url)
+    if GITLAB_PRIVATE_TOKEN:
+        req.add_header('Private-Token', GITLAB_PRIVATE_TOKEN)
+    logger.warn('Performing GitLab API query "%s"' % (url,))
+    return urlopen(req)
+
+
+def _gitlab_api_query(server, path, resource, attrs):
+    url = 'https://%s/api/v4/projects/%s/%s' % (server, urlquote(path, safe=''), resource)
+    if attrs:
+        url += '?' + urlencode(attrs)
+    return _gitlab_urlopen(url)
+
+
+def _gitlab_paged_api_query(server, path, resource, attrs):
+    _attrs = {
+        'per_page': 50,
+        **attrs,
+        'pagination': 'keyset',
+        'page': '1',
+    }
+
+    url = 'https://%s/api/v4/projects/%s/%s' % (server, urlquote(path, safe=''), resource)
+    if _attrs:
+        url += '?' + urlencode(_attrs)
+
+    while True:
+        with _gitlab_urlopen(url) as res:
+            for result in json.loads(res.read().decode('utf-8')):
+                yield result
+
+            # Get the URL to the next page
+            links = res.getheader('Link')
+            if not links:
+                break
+            match = re.match(r'.*<([^>]*)>; rel="next"', links)
+            if not match:
+                break
+            url = match.group(1)
+
+
+def gitlab_manifest_provider(_dist_name, repo, pkg_name):
+    assert repo.version
+    server, path = repo.get_url_parts()
+    if not server.endswith('gitlab.com') and server != ROSDISTRO_GITLAB_SERVER:
+        logger.debug('Skip non-gitlab url "%s"' % repo.url)
+        raise RuntimeError('can not handle non gitlab urls')
+
+    resource = 'repository/files/package.xml/raw'
+    attrs = {
+        'ref': repo.get_release_tag(pkg_name),
+    }
+    try:
+        with _gitlab_api_query(server, path, resource, attrs) as res:
+            return res.read().decode('utf-8')
+    except URLError as e:
+        logger.debug('- failed (%s), trying "%s"' % (e, e.filename))
+        raise
+
+
+def gitlab_source_manifest_provider(repo):
+    assert repo.version
+    server, path = repo.get_url_parts()
+    if not server.endswith('gitlab.com') and server != ROSDISTRO_GITLAB_SERVER:
+        logger.debug('Skip non-gitlab url "%s"' % repo.url)
+        raise RuntimeError('can not handle non gitlab urls')
+
+    # Resolve the version ref to a sha since we need to make multiple queries
+    attrs = {
+        'per_page': 1,
+        'ref_name': repo.version,
+    }
+    sha = next(_gitlab_paged_api_query(server, path, 'repository/commits', attrs))['id']
+
+    # Look for package.xml files in the tree
+    attrs = {
+        'recursive': 'true',
+        'ref': sha,
+    }
+    package_xml_paths = set()
+    for obj in _gitlab_paged_api_query(server, path, 'repository/tree', attrs):
+        if obj['path'].split('/')[-1] == 'package.xml':
+            package_xml_paths.add(os.path.dirname(obj['path']))
+
+    # Filter out ones that are inside other packages (eg, part of tests)
+    def package_xml_in_parent(path):
+        if path == '':
+            return True
+        parent = path
+        while True:
+            parent = os.path.dirname(parent)
+            if parent in package_xml_paths:
+                return False
+            if parent == '':
+                return True
+    package_xml_paths = list(filter(package_xml_in_parent, package_xml_paths))
+
+    cache = SourceRepositoryCache.from_ref(sha)
+    for package_xml_path in package_xml_paths:
+        resource_path = urlquote(
+            package_xml_path + '/package.xml' if package_xml_path else 'package.xml', safe='')
+        resource = 'repository/files/' + resource_path + '/raw'
+        with _gitlab_api_query(server, path, resource, {'ref': sha}) as res:
+            package_xml = res.read().decode('utf-8')
+        name = parse_package_string(package_xml).name
+        cache.add(name, package_xml_path, package_xml)
+
+    return cache

src/rosdistro/release.py

@@ -34,11 +34,12 @@
 from .manifest_provider.bitbucket import bitbucket_manifest_provider
 from .manifest_provider.git import git_manifest_provider
 from .manifest_provider.github import github_manifest_provider
+from .manifest_provider.gitlab import gitlab_manifest_provider
 
 
 class Release(object):
 
-    default_manifest_providers = [github_manifest_provider, bitbucket_manifest_provider, git_manifest_provider]
+    default_manifest_providers = [github_manifest_provider, gitlab_manifest_provider, bitbucket_manifest_provider, git_manifest_provider]
 
     def __init__(self, rel_file, manifest_providers=None):
         self._rel_file = rel_file

test/test_manifest_providers.py

@@ -9,6 +9,7 @@
 from rosdistro.manifest_provider.bitbucket import bitbucket_manifest_provider
 from rosdistro.manifest_provider.cache import CachedManifestProvider, sanitize_xml
 from rosdistro.manifest_provider.git import git_manifest_provider, git_source_manifest_provider
+from rosdistro.manifest_provider.gitlab import gitlab_manifest_provider, gitlab_source_manifest_provider
 from rosdistro.release_repository_specification import ReleaseRepositorySpecification
 from rosdistro.source_repository_specification import SourceRepositorySpecification
 
@@ -17,6 +18,10 @@ def test_bitbucket():
     assert '</package>' in bitbucket_manifest_provider('indigo', _rospeex_release_repo(), 'rospeex_msgs')
 
 
+def test_gitlab():
+    assert '</package>' in gitlab_manifest_provider('foxy', _tracetools_analysis_release_repo(), 'tracetools_analysis')
+
+
 def test_cached():
     class FakeDistributionCache(object):
         def __init__(self):
@@ -97,6 +102,17 @@ def test_github_source():
     assert '<version>0.5.11</version>' in package_xml
 
 
+def test_gitlab_source():
+    repo_cache = gitlab_source_manifest_provider(_tracetools_analysis_source_repo())
+
+    # This hash corresponds to the 1.0.3 tag.
+    assert repo_cache.ref() == 'cd30853005ef3a591cb8594b4aa49f9ef400d30f'
+
+    package_path, package_xml = repo_cache['ros2trace_analysis']
+    assert 'ros2trace_analysis' == package_path
+    assert '<version>1.0.3</version>' in package_xml
+
+
 def test_git_source_multi():
     repo_cache = git_source_manifest_provider(_ros_source_repo())
     assert repo_cache.ref()
@@ -170,3 +186,19 @@ def _rospeex_release_repo():
         'url': 'https://bitbucket.org/rospeex/rospeex-release.git',
         'version': '2.14.7-0'
     })
+
+
+def _tracetools_analysis_release_repo():
+    return ReleaseRepositorySpecification('tracetools_analysis', {
+        'packages': ['ros2trace_analysis', 'tracetools_analysis'],
+        'tags': {'release': 'release/foxy/{package}/{version}'},
+        'url': 'https://gitlab.com/ros-tracing/tracetools_analysis-release.git',
+        'version': '1.0.3-1'
+    })
+
+
+def _tracetools_analysis_source_repo():
+    return SourceRepositorySpecification('tracetools_analysis', {
+        'url': 'https://gitlab.com/ros-tracing/tracetools_analysis.git',
+        'version': '1.0.3'
+    })