Add VSCode extension repos

repology-schemacheck.py

@@ -94,6 +94,7 @@
     'openmandriva',
     'openpkg',
     'opensuse',
+    'openvsx',
     'openwrt',
     'os4depot',
     'pacstall',
@@ -125,6 +126,7 @@
     'termux',
     'ubi',
     'vcpkg',
+    'vscmarketplace',
     'void',
     'wakemeops',
     'wikidata',

repology/fetchers/fetchers/vscmarketplace.py

@@ -0,0 +1,122 @@
+# Copyright (C) 2024 Gavin John <[email protected]>
+#
+# This file is part of repology
+#
+# repology is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# repology is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with repology.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+
+from repology.atomic_fs import AtomicFile
+from repology.fetchers import PersistentData, ScratchFileFetcher
+from repology.fetchers.http import PoliteHTTP
+from repology.logger import Logger
+
+
+class VSCMarketplaceFetcher(ScratchFileFetcher):
+    def __init__(self, page_size: int = 100, fetch_timeout: int = 5, fetch_delay: int | None = None) -> None:
+        self.do_http = PoliteHTTP(timeout=fetch_timeout, delay=fetch_delay)
+
+        self.page_size = page_size
+
+        # Constants
+        self.include_versions = True
+        self.include_files = True
+        self.include_category_and_tags = False
+        self.include_shared_accounts = True
+        self.include_version_properties = True
+        self.exclude_non_validated = False
+        self.include_installation_targets = False
+        self.include_asset_uri = True
+        self.include_statistics = False
+        self.include_latest_version_only = True
+        self.unpublished = False
+        self.include_name_conflict_info = True
+        self.api_version = '7.2-preview.1',
+
+    def _do_fetch(self, statefile: AtomicFile, persdata: PersistentData, logger: Logger) -> bool:
+        extensions = []
+
+        flags = 0
+        if self.include_versions:
+            flags |= 0x1
+
+        if self.include_files:
+            flags |= 0x2
+
+        if self.include_category_and_tags:
+            flags |= 0x4
+
+        if self.include_shared_accounts:
+            flags |= 0x8
+
+        if self.include_version_properties:
+            flags |= 0x10
+
+        if self.exclude_non_validated:
+            flags |= 0x20
+
+        if self.include_installation_targets:
+            flags |= 0x40
+
+        if self.include_asset_uri:
+            flags |= 0x80
+
+        if self.include_statistics:
+            flags |= 0x100
+
+        if self.include_latest_version_only:
+            flags |= 0x200
+
+        if self.unpublished:
+            flags |= 0x1000
+
+        if self.include_name_conflict_info:
+            flags |= 0x8000
+
+        page = 1
+        while True:
+            body = {
+                'filters': [
+                    {
+                        'criteria': [
+                            {
+                                'filterType': 8,
+                                'value': 'Microsoft.VisualStudio.Code'
+                            }
+                        ],
+                        'pageNumber': page,
+                        'pageSize': self.page_size,
+                        'sortBy': 0,
+                        'sortOrder': 0
+                    }
+                ],
+                'assetTypes': [],
+                'flags': flags
+            }
+
+            r = self.do_http('https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery?api-version={version}'.format(version=self.api_version), 'POST', json=body)
+            response = r.json()
+
+            for extension in response['results'][0]['extensions']:
+                extensions.append(extension)
+
+            page += 1
+
+            if len(response['results'][0]['extensions']) < self.page_size:
+                break
+
+        with open(statefile.get_path(), 'w', encoding='utf-8') as extdata:
+            json.dump(extensions, extdata)
+
+        return True

repology/parsers/parsers/openvsx.py

@@ -0,0 +1,43 @@
+# Copyright (C) 2024 Gavin John <[email protected]>
+#
+# This file is part of repology
+#
+# repology is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# repology is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with repology.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+
+from typing import Iterable
+
+from repology.package import LinkType
+from repology.packagemaker import NameType, PackageFactory, PackageMaker
+from repology.parsers import Parser
+
+
+class OpenVSXParser(Parser):
+    def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]:
+        with open(path, 'r') as extdatafile:
+            extension_data = json.load(extdatafile)
+            raw_extensions = extension_data['extensions']
+
+        for extension in raw_extensions:
+            with factory.begin() as pkg:
+                # TODO: More metadata is available, it's just harder to fetch and will require its own fetcher, in all likelihood
+                pkg.add_name('vscode-extension:{namespace}-{name}'.format(**extension), NameType.GENERIC_SRC_NAME)
+                pkg.set_version(extension['version'])
+                pkg.set_summary(extension['description'])
+                pkg.add_maintainers('{namespace}@openvsx'.format(**extension))
+                pkg.add_links(LinkType.UPSTREAM_HOMEPAGE, 'https://open-vsx.org/extension/{namespace}/{name}'.format(**extension))
+                pkg.add_links(LinkType.UPSTREAM_DOWNLOAD, extension['files']['download'])
+
+                yield pkg

repology/parsers/parsers/vscmarketplace.py

@@ -0,0 +1,69 @@
+# Copyright (C) 2024 Gavin John <[email protected]>
+#
+# This file is part of repology
+#
+# repology is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# repology is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with repology.  If not, see <http://www.gnu.org/licenses/>.
+
+import json
+
+from typing import Iterable
+
+from repology.package import LinkType
+from repology.packagemaker import NameType, PackageFactory, PackageMaker
+from repology.parsers import Parser
+
+
+class VSCMarketplaceParser(Parser):
+    def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]:
+        with open(path, 'r') as extdatafile:
+            extension_data = json.load(extdatafile)
+            raw_extensions = extension_data['extensions']
+
+        for extension in raw_extensions:
+            with factory.begin() as pkg:
+                version_idx = 0
+                while True:
+                    for package_property in extension['versions'][version_idx]['properties']:
+                        if package_property['key'] == 'Microsoft.VisualStudio.Code.PreRelease' and package_property['value'] == 'true':
+                            version_idx += 1
+                            continue
+                    break
+
+                pkg.add_name('vscode-extension:{publisherName}-{extensionName}'.format(publisherName=extension['publisher']['publisherName'], extensionName=extension['extensionName']), NameType.GENERIC_SRC_NAME)
+                pkg.set_version(extension['versions'][version_idx]['version'])
+                pkg.set_summary(extension['shortDescription'])
+                pkg.add_maintainers('{publisherName}@vscmarketplace'.format(**extension['publisher']))
+                pkg.add_links(LinkType.UPSTREAM_HOMEPAGE, 'https://marketplace.visualstudio.com/items?itemName={publisherName}.{extensionName}'.format(publisherName=extension['publisher']['publisherName'], extensionName=extension['extensionName']))
+
+                for file_meta in extension['versions'][version_idx]['files']:
+                    match file_meta['assetType']:
+                        case 'Microsoft.VisualStudio.Services.Content.Changelog':
+                            pkg.add_links(LinkType.UPSTREAM_CHANGELOG, file_meta['source'])
+                        case 'Microsoft.VisualStudio.Services.Content.Details':
+                            pkg.add_links(LinkType.UPSTREAM_DOCUMENTATION, file_meta['source'])
+                        case 'Microsoft.VisualStudio.Services.Content.License':
+                            pkg.add_licenses(file_meta['source'])
+
+                for package_property in extension['versions'][version_idx]['properties']:
+                    match package_property['key']:
+                        case 'Microsoft.VisualStudio.Services.Links.Support':
+                            pkg.add_links(LinkType.PACKAGE_ISSUE_TRACKER, package_property['value'])
+                        case 'Microsoft.VisualStudio.Services.Links.Learn':
+                            pkg.add_links(LinkType.PACKAGE_HOMEPAGE, package_property['value'])
+                        case 'Microsoft.VisualStudio.Services.Links.Source':
+                            pkg.add_links(LinkType.PACKAGE_SOURCES, package_property['value'])
+                        case 'Microsoft.VisualStudio.Services.CustomerQnALink':
+                            pkg.add_links(LinkType.UPSTREAM_DISCUSSION, package_property['value'])
+
+                yield pkg

repos.d/openvsx.yaml

@@ -0,0 +1,22 @@
+###########################################################################
+# OpenVSX
+###########################################################################
+- name: openvsx
+  type: repository
+  desc: OpenVSX
+  family: openvsx
+  ruleset: openvsx
+  minpackages: 3000
+  sources:
+    - name: packages
+      fetcher:
+        class: FileFetcher
+        url: https://open-vsx.org/api/-/search?size=10000 # TODO: Do proper pagination
+        allow_zero_size: false
+      parser:
+        class: OpenVSXParser
+  repolinks:
+    - desc: OpenVSX registry
+      url: https://open-vsx.org/
+  packagelinks: []
+  groups: [ all, production ]

repos.d/vscmarketplace.yaml

@@ -0,0 +1,20 @@
+###########################################################################
+# VSC Marketplace
+###########################################################################
+- name: vscmarketplace
+  type: repository
+  desc: Visual Studio Code Marketplace
+  family: vscmarketplace
+  ruleset: vscmarketplace
+  minpackages: 3000
+  sources:
+    - name: packages
+      fetcher:
+        class: VSCMarketplaceFetcher
+      parser:
+        class: VSCMarketplaceParser
+  repolinks:
+    - desc: Visual Studio Code Marketplace
+      url: https://marketplace.visualstudio.com/vscode
+  packagelinks: []
+  groups: [ all, production ]