diff --git a/.ci/scripts/auditwheel_wrapper.py b/.ci/scripts/auditwheel_wrapper.py deleted file mode 100755 index 98328212217..00000000000 --- a/.ci/scripts/auditwheel_wrapper.py +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env python -# -# This file is licensed under the Affero General Public License (AGPL) version 3. -# -# Copyright (C) 2023 New Vector, Ltd -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# See the GNU Affero General Public License for more details: -# . -# -# Originally licensed under the Apache License, Version 2.0: -# . -# -# [This file includes modifications made by New Vector Limited] -# -# - -# Wraps `auditwheel repair` to first check if we're repairing a potentially abi3 -# compatible wheel, if so rename the wheel before repairing it. - -import argparse -import os -import subprocess -from zipfile import ZipFile - -from packaging.tags import Tag -from packaging.utils import parse_wheel_filename -from packaging.version import Version - - -def check_is_abi3_compatible(wheel_file: str) -> None: - """Check the contents of the built wheel for any `.so` files that are *not* - abi3 compatible. - """ - - with ZipFile(wheel_file, "r") as wheel: - for file in wheel.namelist(): - if not file.endswith(".so"): - continue - - if not file.endswith(".abi3.so"): - raise Exception(f"Found non-abi3 lib: {file}") - - -def cpython(wheel_file: str, name: str, version: Version, tag: Tag) -> str: - """Replaces the cpython wheel file with a ABI3 compatible wheel""" - - if tag.abi == "abi3": - # Nothing to do. - return wheel_file - - check_is_abi3_compatible(wheel_file) - - # HACK: it seems that some older versions of pip will consider a wheel marked - # as macosx_11_0 as incompatible with Big Sur. I haven't done the full archaeology - # here; there are some clues in - # https://github.com/pantsbuild/pants/pull/12857 - # https://github.com/pypa/pip/issues/9138 - # https://github.com/pypa/packaging/pull/319 - # Empirically this seems to work, note that macOS 11 and 10.16 are the same, - # both versions are valid for backwards compatibility. - platform = tag.platform.replace("macosx_11_0", "macosx_10_16") - abi3_tag = Tag(tag.interpreter, "abi3", platform) - - dirname = os.path.dirname(wheel_file) - new_wheel_file = os.path.join( - dirname, - f"{name}-{version}-{abi3_tag}.whl", - ) - - os.rename(wheel_file, new_wheel_file) - - print("Renamed wheel to", new_wheel_file) - - return new_wheel_file - - -def main(wheel_file: str, dest_dir: str, archs: str | None) -> None: - """Entry point""" - - # Parse the wheel file name into its parts. Note that `parse_wheel_filename` - # normalizes the package name (i.e. it converts matrix_synapse -> - # matrix-synapse), which is not what we want. - _, version, build, tags = parse_wheel_filename(os.path.basename(wheel_file)) - name = os.path.basename(wheel_file).split("-")[0] - - if len(tags) != 1: - # We expect only a wheel file with only a single tag - raise Exception(f"Unexpectedly found multiple tags: {tags}") - - tag = next(iter(tags)) - - if build: - # We don't use build tags in Synapse - raise Exception(f"Unexpected build tag: {build}") - - # If the wheel is for cpython then convert it into an abi3 wheel. - if tag.interpreter.startswith("cp"): - wheel_file = cpython(wheel_file, name, version, tag) - - # Finally, repair the wheel. - if archs is not None: - # If we are given archs then we are on macos and need to use - # `delocate-listdeps`. - subprocess.run(["delocate-listdeps", wheel_file], check=True) - subprocess.run( - ["delocate-wheel", "--require-archs", archs, "-w", dest_dir, wheel_file], - check=True, - ) - else: - subprocess.run(["auditwheel", "repair", "-w", dest_dir, wheel_file], check=True) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Tag wheel as abi3 and repair it.") - - parser.add_argument( - "--wheel-dir", - "-w", - metavar="WHEEL_DIR", - help="Directory to store delocated wheels", - required=True, - ) - - parser.add_argument( - "--require-archs", - metavar="archs", - default=None, - ) - - parser.add_argument( - "wheel_file", - metavar="WHEEL_FILE", - ) - - args = parser.parse_args() - - wheel_file = args.wheel_file - wheel_dir = args.wheel_dir - archs = args.require_archs - - main(wheel_file, wheel_dir, archs) diff --git a/changelog.d/19234.misc b/changelog.d/19234.misc new file mode 100644 index 00000000000..d79bc0b19fc --- /dev/null +++ b/changelog.d/19234.misc @@ -0,0 +1 @@ +Switch the build backend from `poetry-core` to `maturin`. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 5ee843365d5..28f28d0f682 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -120,7 +120,7 @@ redis = ["txredisapi>=1.4.7", "hiredis"] # Required to use experimental `caches.track_memory_usage` config option. cache-memory = ["pympler"] # If this is updated, don't forget to update the equivalent lines in -# tool.poetry.group.dev.dependencies. +# `dependency-groups.dev` below. test = ["parameterized>=0.9.0", "idna>=3.3"] # The duplication here is awful. @@ -177,6 +177,64 @@ synapse_port_db = "synapse._scripts.synapse_port_db:main" synapse_review_recent_signups = "synapse._scripts.review_recent_signups:main" update_synapse_database = "synapse._scripts.update_synapse_database:main" +[tool.poetry] +packages = [{ include = "synapse" }] + +[tool.poetry.build] +script = "build_rust.py" +generate-setup-file = true + +# Dependencies used for developing Synapse itself. +# +# Hold off on migrating these to `dev-dependencies` (PEP 735) for now until +# Poetry 2.2.0+, pip 25.1+ are more widely available. +[tool.poetry.group.dev.dependencies] +# We pin development dependencies in poetry.lock so that our tests don't start +# failing on new releases. Keeping lower bounds loose here means that dependabot +# can bump versions without having to update the content-hash in the lockfile. +# This helps prevents merge conflicts when running a batch of dependabot updates. +ruff = "0.14.5" + +# Typechecking +lxml-stubs = ">=0.4.0" +mypy = "*" +mypy-zope = "*" +types-bleach = ">=4.1.0" +types-jsonschema = ">=3.2.0" +types-netaddr = ">=0.8.0.6" +types-opentracing = ">=2.4.2" +types-Pillow = ">=8.3.4" +types-psycopg2 = ">=2.9.9" +types-pyOpenSSL = ">=20.0.7" +types-PyYAML = ">=5.4.10" +types-requests = ">=2.26.0" +types-setuptools = ">=57.4.0" + +# Dependencies which are exclusively required by unit test code. This is +# NOT a list of all modules that are necessary to run the unit tests. +# Tests assume that all optional dependencies are installed. +# +# If this is updated, don't forget to update the equivalent lines in +# project.optional-dependencies.test. +parameterized = ">=0.9.0" +idna = ">=3.3" + +# The following are used by the release script +click = ">=8.1.3" +# GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints. +GitPython = ">=3.1.20" +markdown-it-py = ">=3.0.0" +pygithub = ">=1.59" +# The following are executed as commands by the release script. +twine = "*" +# Towncrier min version comes from https://github.com/matrix-org/synapse/pull/3425. Rationale unclear. +towncrier = ">=18.6.0rc1" + +# Used for checking the Poetry lockfile +tomli = ">=1.2.3" + +# Used for checking the schema delta files +sqlglot = ">=28.0.0" [tool.towncrier] package = "synapse" @@ -291,88 +349,29 @@ line-ending = "auto" [tool.maturin] manifest-path = "rust/Cargo.toml" module-name = "synapse.synapse_rust" - -[tool.poetry] -packages = [ - { include = "synapse" }, -] -include = [ - { path = "AUTHORS.rst", format = "sdist" }, - { path = "book.toml", format = "sdist" }, - { path = "changelog.d", format = "sdist" }, - { path = "CHANGES.md", format = "sdist" }, - { path = "CONTRIBUTING.md", format = "sdist" }, - { path = "demo", format = "sdist" }, - { path = "docs", format = "sdist" }, - { path = "INSTALL.md", format = "sdist" }, - { path = "mypy.ini", format = "sdist" }, - { path = "scripts-dev", format = "sdist" }, - { path = "synmark", format="sdist" }, - { path = "sytest-blacklist", format = "sdist" }, - { path = "tests", format = "sdist" }, - { path = "UPGRADE.rst", format = "sdist" }, - { path = "Cargo.toml", format = "sdist" }, - { path = "Cargo.lock", format = "sdist" }, - { path = "rust/Cargo.toml", format = "sdist" }, - { path = "rust/build.rs", format = "sdist" }, - { path = "rust/src/**", format = "sdist" }, +python-source = "." +sdist-include = [ + "AUTHORS.rst", + "book.toml", + "changelog.d", + "CHANGES.md", + "CONTRIBUTING.md", + "demo", + "docs", + "INSTALL.md", + "mypy.ini", + "scripts-dev", + "synmark", + "sytest-blacklist", + "tests", + "UPGRADE.rst", + "Cargo.toml", + "Cargo.lock", + "rust/Cargo.toml", + "rust/build.rs", + "rust/src/**", ] -exclude = [ - { path = "synapse/*.so", format = "sdist"} -] - -[tool.poetry.build] -script = "build_rust.py" -generate-setup-file = true - -[tool.poetry.group.dev.dependencies] -# We pin development dependencies in poetry.lock so that our tests don't start -# failing on new releases. Keeping lower bounds loose here means that dependabot -# can bump versions without having to update the content-hash in the lockfile. -# This helps prevents merge conflicts when running a batch of dependabot updates. -ruff = "0.14.5" - -# Typechecking -lxml-stubs = ">=0.4.0" -mypy = "*" -mypy-zope = "*" -types-bleach = ">=4.1.0" -types-jsonschema = ">=3.2.0" -types-netaddr = ">=0.8.0.6" -types-opentracing = ">=2.4.2" -types-Pillow = ">=8.3.4" -types-psycopg2 = ">=2.9.9" -types-pyOpenSSL = ">=20.0.7" -types-PyYAML = ">=5.4.10" -types-requests = ">=2.26.0" -types-setuptools = ">=57.4.0" - -# Dependencies which are exclusively required by unit test code. This is -# NOT a list of all modules that are necessary to run the unit tests. -# Tests assume that all optional dependencies are installed. -# -# If this is updated, don't forget to update the equivalent lines in -# project.optional-dependencies.test. -parameterized = ">=0.9.0" -idna = ">=3.3" - -# The following are used by the release script -click = ">=8.1.3" -# GitPython was == 3.1.14; bumped to 3.1.20, the first release with type hints. -GitPython = ">=3.1.20" -markdown-it-py = ">=3.0.0" -pygithub = ">=1.59" -# The following are executed as commands by the release script. -twine = "*" -# Towncrier min version comes from https://github.com/matrix-org/synapse/pull/3425. Rationale unclear. -towncrier = ">=18.6.0rc1" - -# Used for checking the Poetry lockfile -tomli = ">=1.2.3" - -# Used for checking the schema delta files -sqlglot = ">=28.0.0" - +sdist-exclude = ["synapse/*.so"] [build-system] # The upper bounds here are defensive, intended to prevent situations like @@ -381,8 +380,8 @@ sqlglot = ">=28.0.0" # runtime errors caused by build system changes. # We are happy to raise these upper bounds upon request, # provided we check that it's safe to do so (i.e. that CI passes). -requires = ["poetry-core>=2.0.0,<=2.1.3", "setuptools_rust>=1.3,<=1.11.1"] -build-backend = "poetry.core.masonry.api" +requires = ["maturin>=1.0,<2.0"] +build-backend = "maturin" [tool.cibuildwheel] @@ -419,12 +418,3 @@ environment= { PATH = "$PATH:$HOME/.cargo/bin" } before-build = "rm -rf {project}/build" build-frontend = "build" test-command = "python -c 'from synapse.synapse_rust import sum_as_string; print(sum_as_string(1, 2))'" - - -[tool.cibuildwheel.linux] -# Wrap the repair command to correctly rename the built cpython wheels as ABI3. -repair-wheel-command = "./.ci/scripts/auditwheel_wrapper.py -w {dest_dir} {wheel}" - -[tool.cibuildwheel.macos] -# Wrap the repair command to correctly rename the built cpython wheels as ABI3. -repair-wheel-command = "./.ci/scripts/auditwheel_wrapper.py --require-archs {delocate_archs} -w {dest_dir} {wheel}"