diff --git a/repology-schemacheck.py b/repology-schemacheck.py index 91bd0c14a..8080a67c7 100755 --- a/repology-schemacheck.py +++ b/repology-schemacheck.py @@ -94,6 +94,7 @@ 'openmandriva', 'openpkg', 'opensuse', + 'openvsx', 'openwrt', 'os4depot', 'pacstall', @@ -125,6 +126,7 @@ 'termux', 'ubi', 'vcpkg', + 'vscmarketplace', 'void', 'wakemeops', 'wikidata', diff --git a/repology/fetchers/fetchers/vscmarketplace.py b/repology/fetchers/fetchers/vscmarketplace.py new file mode 100644 index 000000000..7dbc96ed3 --- /dev/null +++ b/repology/fetchers/fetchers/vscmarketplace.py @@ -0,0 +1,122 @@ +# Copyright (C) 2024 Gavin John +# +# This file is part of repology +# +# repology is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# repology is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with repology. If not, see . + +import json + +from repology.atomic_fs import AtomicFile +from repology.fetchers import PersistentData, ScratchFileFetcher +from repology.fetchers.http import PoliteHTTP +from repology.logger import Logger + + +class VSCMarketplaceFetcher(ScratchFileFetcher): + def __init__(self, page_size: int = 100, fetch_timeout: int = 5, fetch_delay: int | None = None) -> None: + self.do_http = PoliteHTTP(timeout=fetch_timeout, delay=fetch_delay) + + self.page_size = page_size + + # Constants + self.include_versions = True + self.include_files = True + self.include_category_and_tags = False + self.include_shared_accounts = True + self.include_version_properties = True + self.exclude_non_validated = False + self.include_installation_targets = False + self.include_asset_uri = True + self.include_statistics = False + self.include_latest_version_only = True + self.unpublished = False + self.include_name_conflict_info = True + self.api_version = '7.2-preview.1', + + def _do_fetch(self, statefile: AtomicFile, persdata: PersistentData, logger: Logger) -> bool: + extensions = [] + + flags = 0 + if self.include_versions: + flags |= 0x1 + + if self.include_files: + flags |= 0x2 + + if self.include_category_and_tags: + flags |= 0x4 + + if self.include_shared_accounts: + flags |= 0x8 + + if self.include_version_properties: + flags |= 0x10 + + if self.exclude_non_validated: + flags |= 0x20 + + if self.include_installation_targets: + flags |= 0x40 + + if self.include_asset_uri: + flags |= 0x80 + + if self.include_statistics: + flags |= 0x100 + + if self.include_latest_version_only: + flags |= 0x200 + + if self.unpublished: + flags |= 0x1000 + + if self.include_name_conflict_info: + flags |= 0x8000 + + page = 1 + while True: + body = { + 'filters': [ + { + 'criteria': [ + { + 'filterType': 8, + 'value': 'Microsoft.VisualStudio.Code' + } + ], + 'pageNumber': page, + 'pageSize': self.page_size, + 'sortBy': 0, + 'sortOrder': 0 + } + ], + 'assetTypes': [], + 'flags': flags + } + + r = self.do_http('https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery?api-version={version}'.format(version=self.api_version), 'POST', json=body) + response = r.json() + + for extension in response['results'][0]['extensions']: + extensions.append(extension) + + page += 1 + + if len(response['results'][0]['extensions']) < self.page_size: + break + + with open(statefile.get_path(), 'w', encoding='utf-8') as extdata: + json.dump(extensions, extdata) + + return True diff --git a/repology/parsers/parsers/openvsx.py b/repology/parsers/parsers/openvsx.py new file mode 100644 index 000000000..4a3cc2958 --- /dev/null +++ b/repology/parsers/parsers/openvsx.py @@ -0,0 +1,43 @@ +# Copyright (C) 2024 Gavin John +# +# This file is part of repology +# +# repology is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# repology is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with repology. If not, see . + +import json + +from typing import Iterable + +from repology.package import LinkType +from repology.packagemaker import NameType, PackageFactory, PackageMaker +from repology.parsers import Parser + + +class OpenVSXParser(Parser): + def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]: + with open(path, 'r') as extdatafile: + extension_data = json.load(extdatafile) + raw_extensions = extension_data['extensions'] + + for extension in raw_extensions: + with factory.begin() as pkg: + # TODO: More metadata is available, it's just harder to fetch and will require its own fetcher, in all likelihood + pkg.add_name('vscode-extension:{namespace}-{name}'.format(**extension), NameType.GENERIC_SRC_NAME) + pkg.set_version(extension['version']) + pkg.set_summary(extension['description']) + pkg.add_maintainers('{namespace}@openvsx'.format(**extension)) + pkg.add_links(LinkType.UPSTREAM_HOMEPAGE, 'https://open-vsx.org/extension/{namespace}/{name}'.format(**extension)) + pkg.add_links(LinkType.UPSTREAM_DOWNLOAD, extension['files']['download']) + + yield pkg diff --git a/repology/parsers/parsers/vscmarketplace.py b/repology/parsers/parsers/vscmarketplace.py new file mode 100644 index 000000000..bc89b5fcc --- /dev/null +++ b/repology/parsers/parsers/vscmarketplace.py @@ -0,0 +1,69 @@ +# Copyright (C) 2024 Gavin John +# +# This file is part of repology +# +# repology is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# repology is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with repology. If not, see . + +import json + +from typing import Iterable + +from repology.package import LinkType +from repology.packagemaker import NameType, PackageFactory, PackageMaker +from repology.parsers import Parser + + +class VSCMarketplaceParser(Parser): + def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]: + with open(path, 'r') as extdatafile: + extension_data = json.load(extdatafile) + raw_extensions = extension_data['extensions'] + + for extension in raw_extensions: + with factory.begin() as pkg: + version_idx = 0 + while True: + for package_property in extension['versions'][version_idx]['properties']: + if package_property['key'] == 'Microsoft.VisualStudio.Code.PreRelease' and package_property['value'] == 'true': + version_idx += 1 + continue + break + + pkg.add_name('vscode-extension:{publisherName}-{extensionName}'.format(publisherName=extension['publisher']['publisherName'], extensionName=extension['extensionName']), NameType.GENERIC_SRC_NAME) + pkg.set_version(extension['versions'][version_idx]['version']) + pkg.set_summary(extension['shortDescription']) + pkg.add_maintainers('{publisherName}@vscmarketplace'.format(**extension['publisher'])) + pkg.add_links(LinkType.UPSTREAM_HOMEPAGE, 'https://marketplace.visualstudio.com/items?itemName={publisherName}.{extensionName}'.format(publisherName=extension['publisher']['publisherName'], extensionName=extension['extensionName'])) + + for file_meta in extension['versions'][version_idx]['files']: + match file_meta['assetType']: + case 'Microsoft.VisualStudio.Services.Content.Changelog': + pkg.add_links(LinkType.UPSTREAM_CHANGELOG, file_meta['source']) + case 'Microsoft.VisualStudio.Services.Content.Details': + pkg.add_links(LinkType.UPSTREAM_DOCUMENTATION, file_meta['source']) + case 'Microsoft.VisualStudio.Services.Content.License': + pkg.add_licenses(file_meta['source']) + + for package_property in extension['versions'][version_idx]['properties']: + match package_property['key']: + case 'Microsoft.VisualStudio.Services.Links.Support': + pkg.add_links(LinkType.PACKAGE_ISSUE_TRACKER, package_property['value']) + case 'Microsoft.VisualStudio.Services.Links.Learn': + pkg.add_links(LinkType.PACKAGE_HOMEPAGE, package_property['value']) + case 'Microsoft.VisualStudio.Services.Links.Source': + pkg.add_links(LinkType.PACKAGE_SOURCES, package_property['value']) + case 'Microsoft.VisualStudio.Services.CustomerQnALink': + pkg.add_links(LinkType.UPSTREAM_DISCUSSION, package_property['value']) + + yield pkg diff --git a/repos.d/openvsx.yaml b/repos.d/openvsx.yaml new file mode 100644 index 000000000..61b88387b --- /dev/null +++ b/repos.d/openvsx.yaml @@ -0,0 +1,22 @@ +########################################################################### +# OpenVSX +########################################################################### +- name: openvsx + type: repository + desc: OpenVSX + family: openvsx + ruleset: openvsx + minpackages: 3000 + sources: + - name: packages + fetcher: + class: FileFetcher + url: https://open-vsx.org/api/-/search?size=10000 # TODO: Do proper pagination + allow_zero_size: false + parser: + class: OpenVSXParser + repolinks: + - desc: OpenVSX registry + url: https://open-vsx.org/ + packagelinks: [] + groups: [ all, production ] diff --git a/repos.d/vscmarketplace.yaml b/repos.d/vscmarketplace.yaml new file mode 100644 index 000000000..183dd73a0 --- /dev/null +++ b/repos.d/vscmarketplace.yaml @@ -0,0 +1,20 @@ +########################################################################### +# VSC Marketplace +########################################################################### +- name: vscmarketplace + type: repository + desc: Visual Studio Code Marketplace + family: vscmarketplace + ruleset: vscmarketplace + minpackages: 3000 + sources: + - name: packages + fetcher: + class: VSCMarketplaceFetcher + parser: + class: VSCMarketplaceParser + repolinks: + - desc: Visual Studio Code Marketplace + url: https://marketplace.visualstudio.com/vscode + packagelinks: [] + groups: [ all, production ]