diff --git a/core-requirements.txt b/core-requirements.txt index 5d7e7ad..ac26340 100644 --- a/core-requirements.txt +++ b/core-requirements.txt @@ -5,3 +5,4 @@ wheel-filename wheel-inspect>=1.6.0 pip>=1.5.3 junit-xml +pip-requirements-parser>=32.0.1 diff --git a/devpi_builder/cli.py b/devpi_builder/cli.py index b018a89..e367b17 100644 --- a/devpi_builder/cli.py +++ b/devpi_builder/cli.py @@ -114,7 +114,8 @@ def build_packages(self, packages): for package, version in packages: if self._should_package_be_build(package, version): - logger.info('Building %s %s', package, version) + msg = ('Building %s %s', package, version) if version else ('Building %s', package) + logger.info(*msg) try: wheel_file = self._builder(package, version) self._upload_package(package, version, wheel_file) diff --git a/devpi_builder/requirements.py b/devpi_builder/requirements.py index 6132ef8..179998a 100644 --- a/devpi_builder/requirements.py +++ b/devpi_builder/requirements.py @@ -4,6 +4,7 @@ Functionality for reading specifications of required packages. """ +import pip_requirements_parser import pkg_resources @@ -14,23 +15,25 @@ def _extract_project_version(requirement): :param requirement: A pkg_config Requirement :return: Pair of project_name and version """ - specs = requirement.specs - if len(specs) == 1: - spec = specs[0] - if spec[0] == '==': - return requirement.project_name, spec[1] - else: - raise ValueError('Versions must be specified exactly. "{}" is not an exact version specification.'.format(requirement)) - elif len(specs) > 1: - raise ValueError('Multiple version specifications on a single line are not supported.') + if requirement.is_vcs_url: + return requirement.line, None + if not requirement.specifier: + raise ValueError('Version specification is missing for "{}".'.format(requirement.line)) else: - raise ValueError('Version specification is missing for "{}".'.format(requirement)) + if len(requirement.specifier) == 1 and requirement.is_pinned: + return requirement.name, requirement.get_pinned_version + elif len(requirement.specifier) == 1 and not requirement.is_pinned: + raise ValueError('Versions must be specified exactly. "{}" is not an exact version specification.'.format(requirement.line)) + elif len(requirement.specifier) > 1: + raise ValueError('Multiple version specifications on a single line are not supported.') def read_raw(filename): if filename: - with open(filename) as requirements_file: - return list(pkg_resources.parse_requirements(requirements_file)) + rf = pip_requirements_parser.RequirementsFile.from_file(filename) + if rf.invalid_lines: + raise ValueError("There are invalid lines in requirements file: \n", "\n".join(line.dumps() for line in rf.invalid_lines)) + return rf.requirements else: return [] @@ -61,7 +64,7 @@ def matched_by_list(package, version, requirements): version = pkg_resources.safe_version('{}'.format(version)) package = pkg_resources.safe_name(package) matches = ( - package.lower() == requirement.key and version in requirement + package.lower() == pkg_resources.safe_name(requirement.name) and (requirement.specifier.contains(version) if requirement.specifier else 1) for requirement in requirements ) return any(matches) diff --git a/devpi_builder/wheeler.py b/devpi_builder/wheeler.py index e91d48c..66bc621 100644 --- a/devpi_builder/wheeler.py +++ b/devpi_builder/wheeler.py @@ -19,6 +19,7 @@ class BuildError(Exception): def __init__(self, package, version, root_exception=None): + version = version or "" super(BuildError, self).__init__('Failed to create wheel for {} {}:\n{}\nOutput:\n{}'.format( package, version, @@ -40,6 +41,10 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): shutil.rmtree(self.scratch_dir) + @staticmethod + def _standardize_package_name(name): + return name.replace(".", "-") + def _matches_requirement(self, requirement, wheels): """ List wheels matching a requirement. @@ -52,7 +57,7 @@ def _matches_requirement(self, requirement, wheels): matching = [] for wheel in wheels: w = wheel.parsed_filename - dist = Distribution(project_name=w.project, version=w.version) + dist = Distribution(project_name=self._standardize_package_name(w.project), version=w.version) if dist in req: matching.append(wheel.path) return matching @@ -63,7 +68,9 @@ def _find_wheel(self, name, version): Find a wheel with the given name and version """ candidates = [WheelFile(filename) for filename in glob.iglob(path.join(self.wheelhouse, '*.whl'))] - matches = self._matches_requirement('{}=={}'.format(name, version), candidates) + name = self._standardize_package_name(name) + requirement = '{}=={}'.format(name, version) if version else name + matches = self._matches_requirement(requirement, candidates) if len(matches) > 0: return str(matches[0]) else: @@ -81,7 +88,7 @@ def build(self, package, version): subprocess.check_output([ 'pip', 'wheel', '--wheel-dir=' + self.wheelhouse, - '{}=={}'.format(package, version) + '{}=={}'.format(package, version) if version else package, ], stderr=subprocess.STDOUT) return self._find_wheel(package, version) except subprocess.CalledProcessError as e: diff --git a/requirements.txt b/requirements.txt index 6aeff28..ad6375e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.7 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # pip-compile --no-emit-index-url @@ -93,6 +93,7 @@ multidict==6.0.2 packaging==21.3 # via # build + # pip-requirements-parser # pytest # setuptools-scm # tox @@ -103,6 +104,8 @@ pastedeploy==2.1.1 # via plaster-pastedeploy pep517==0.12.0 # via build +pip-requirements-parser==32.0.1 + # via -r core-requirements.txt pkginfo==1.8.2 # via devpi-client plaster==1.0 @@ -133,7 +136,9 @@ pycparser==2.21 pygments==2.15.0 # via readme-renderer pyparsing==3.0.9 - # via packaging + # via + # packaging + # pip-requirements-parser pyramid==2.0.2 # via devpi-server pytest==7.1.2 @@ -157,6 +162,8 @@ requests==2.31.0 # devpi-plumber ruamel-yaml==0.17.21 # via devpi-server +ruamel-yaml-clib==0.2.7 + # via ruamel-yaml setuptools-scm==6.4.2 # via -r requirements.in six==1.16.0 @@ -173,7 +180,9 @@ toml==0.10.2 # via tox tomli==2.0.1 # via + # build # check-manifest + # coverage # pep517 # pytest # setuptools-scm diff --git a/tests/fixture/sample_vcs.txt b/tests/fixture/sample_vcs.txt new file mode 100644 index 0000000..db8a868 --- /dev/null +++ b/tests/fixture/sample_vcs.txt @@ -0,0 +1,3 @@ +requests @ git+https://github.com/kennethreitz/requests@master +requests @ git+ssh://github.com/kennethreitz/requests@master +requests @ git+https://github.com/kennethreitz/requests@2.16.0 diff --git a/tests/test_cli.py b/tests/test_cli.py index 2d62a3d..396c4c9 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -162,7 +162,7 @@ def _assert_junit_xml_content(junit_filename, run_id=None): root = ET.parse(junit_filename) ET.dump(root) - _assert_test_case(root, 'failure', 'package-that-hopefully-not-exists 99.999' + run_id_str) + _assert_test_case(root, 'failure', 'package_that_hopefully_not_exists 99.999' + run_id_str) _assert_test_case(root, 'skipped', 'test-package 0.1.dev1' + run_id_str) pb_elems = root.findall(".//testcase[@name='progressbar 2.2{}']".format(run_id_str)) diff --git a/tests/test_requirements.py b/tests/test_requirements.py index 798ac74..cb94f19 100644 --- a/tests/test_requirements.py +++ b/tests/test_requirements.py @@ -8,10 +8,19 @@ def test_read_requirements(): expected = [ ('progressbar', '2.2'), - ('six', '1.7.3') + ('six', '1.7.3'), ] assert expected == requirements.read_exact_versions('tests/fixture/sample_simple.txt') +def test_read_vcs_requirements(): + expected = [ + ('requests @ git+https://github.com/kennethreitz/requests@master', None), + ('requests @ git+ssh://github.com/kennethreitz/requests@master', None), + ('requests @ git+https://github.com/kennethreitz/requests@2.16.0', None), + ] + assert expected == requirements.read_exact_versions('tests/fixture/sample_vcs.txt') + + def test_multiple_versions(): expected = [ diff --git a/tests/test_wheeler.py b/tests/test_wheeler.py index 20ed646..1a70456 100644 --- a/tests/test_wheeler.py +++ b/tests/test_wheeler.py @@ -9,9 +9,19 @@ class TestBuilder: - def test_build(self): + @pytest.mark.parametrize( + 'args', ( + ('progressbar', '2.2'), + ('progressbar', None), + ('requests @ git+https://github.com/kennethreitz/requests@2.16.0', None), + ('requests @ git+https://github.com/kennethreitz/requests@master', None), + ('zope-event', '5.0'), + ('devpi-common', '3.6.0'), + ) + ) + def test_build(self, args): with wheeler.Builder() as builder: - wheel_file = builder('progressbar', '2.2') + wheel_file = builder(*args) assert re.match(r'.*\.whl$', wheel_file) assert path.exists(wheel_file)