Update GitLab manifest provider to support auth, self hosting

Author: cottsay

src/rosdistro/manifest_provider/gitlab.py

@@ -31,37 +31,49 @@
 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-try:
-    from urllib.request import urlopen, Request
-    from urllib.error import URLError
-    from urllib.parse import quote as urlquote
-except ImportError:
-    from urllib2 import urlopen, Request
-    from urllib2 import URLError
-    from urllib2.parse import quote as urlquote
-
 import json
 import os
 import re
+from urllib.request import urlopen, Request
+from urllib.error import URLError
+from urllib.parse import quote as urlquote
 
 from catkin_pkg.package import parse_package_string
 
 from rosdistro.source_repository_cache import SourceRepositoryCache
 from rosdistro import logger
 
+GITLAB_PRIVATE_TOKEN = os.getenv('GITLAB_PRIVATE_TOKEN', None)
+
+def _gitlab_urlopen(url):
+    req = Request(url)
+    if GITLAB_PRIVATE_TOKEN:
+        req.add_header('Private-Token', GITLAB_PRIVATE_TOKEN)
+    logger.warn('Performing GitLab API query "%s"' % (url,))
+    return urlopen(req)
+
+
+def _gitlab_api_query(server, path, resource, attrs):
+    url = 'https://%s/api/v4/projects/%s/%s' % (server, urlquote(path, safe=''), resource)
+    if attrs:
+        url += '?' + '&'.join(k + '=' + urlquote(str(v), safe='') for k, v in attrs.items())
+    return _gitlab_urlopen(url)
 
-def _gitlab_paged_api_query(path, resource, attrs):
-    _attrs = {'per_page': 50}
-    _attrs.update(attrs)
-    _attrs.pop('pagination', None)
-    _attrs.pop('page', None)
 
-    url = 'https://gitlab.com/api/v4/projects/%s/%s?pagination=keyset' % (urlquote(path, safe=''), resource)
-    for k, v in _attrs.items():
-        url += '&%s=%s' % (k, urlquote(str(v), safe=''))
+def _gitlab_paged_api_query(server, path, resource, attrs):
+    _attrs = {
+        'per_page': 50,
+        **attrs,
+        'pagination': 'keyset',
+        'page': '1',
+    }
+
+    url = 'https://%s/api/v4/projects/%s/%s' % (server, urlquote(path, safe=''), resource)
+    if _attrs:
+        url += '?' + '&'.join(k + '=' + urlquote(str(v), safe='') for k, v in _attrs.items())
 
     while True:
-        with urlopen(url) as res:
+        with _gitlab_urlopen(url) as res:
             for result in json.loads(res.read().decode('utf-8')):
                 yield result
 
@@ -78,37 +90,43 @@ def _gitlab_paged_api_query(path, resource, attrs):
 def gitlab_manifest_provider(_dist_name, repo, pkg_name):
     assert repo.version
     server, path = repo.get_url_parts()
-    if not server.endswith('gitlab.com'):
+    if not server.startswith('gitlab.'):
         logger.debug('Skip non-gitlab url "%s"' % repo.url)
         raise RuntimeError('can not handle non gitlab urls')
 
-    release_tag = repo.get_release_tag(pkg_name)
-
-    if not repo.has_remote_tag(release_tag):
-        raise RuntimeError('specified tag "%s" is not a git tag' % release_tag)
-
-    url = 'https://gitlab.com/%s/-/raw/%s/package.xml' % (path, release_tag)
+    resource = 'repository/files/package.xml/raw'
+    attrs = {
+        'ref': repo.get_release_tag(pkg_name),
+    }
     try:
-        logger.debug('Load package.xml file from url "%s"' % url)
-        return urlopen(url).read().decode('utf-8')
+        with _gitlab_api_query(server, path, resource, attrs) as res:
+            return res.read().decode('utf-8')
     except URLError as e:
-        logger.debug('- failed (%s), trying "%s"' % (e, url))
-        raise RuntimeError()
+        logger.debug('- failed (%s), trying "%s"' % (e, e.filename))
+        raise
 
 
 def gitlab_source_manifest_provider(repo):
     assert repo.version
     server, path = repo.get_url_parts()
-    if not server.endswith('gitlab.com'):
+    if not server.startswith('gitlab.'):
         logger.debug('Skip non-gitlab url "%s"' % repo.url)
         raise RuntimeError('can not handle non gitlab urls')
 
-    # Resolve the version ref to a sha
-    sha = next(_gitlab_paged_api_query(path, 'repository/commits', {'per_page': 1, 'ref_name': repo.version}))['id']
+    # Resolve the version ref to a sha since we need to make multiple queries
+    attrs = {
+        'per_page': 1,
+        'ref_name': repo.version,
+    }
+    sha = next(_gitlab_paged_api_query(server, path, 'repository/commits', attrs))['id']
 
     # Look for package.xml files in the tree
+    attrs = {
+        'recursive': 'true',
+        'ref': sha,
+    }
     package_xml_paths = set()
-    for obj in _gitlab_paged_api_query(path, 'repository/tree', {'recursive': 'true', 'ref': sha}):
+    for obj in _gitlab_paged_api_query(server, path, 'repository/tree', attrs):
         if obj['path'].split('/')[-1] == 'package.xml':
             package_xml_paths.add(os.path.dirname(obj['path']))
 
@@ -127,10 +145,11 @@ def package_xml_in_parent(path):
 
     cache = SourceRepositoryCache.from_ref(sha)
     for package_xml_path in package_xml_paths:
-        url = 'https://gitlab.com/%s/-/raw/%s/%s' % \
-            (path, sha, package_xml_path + '/package.xml' if package_xml_path else 'package.xml')
-        logger.debug('- load package.xml from %s' % url)
-        package_xml = urlopen(url).read().decode('utf-8')
+        resource_path = urlquote(
+            package_xml_path + '/package.xml' if package_xml_path else 'package.xml', safe='')
+        resource = 'repository/files/' + resource_path + '/raw'
+        with _gitlab_api_query(server, path, resource, {'ref': sha}) as res:
+            package_xml = res.read().decode('utf-8')
         name = parse_package_string(package_xml).name
         cache.add(name, package_xml_path, package_xml)