diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 229bce922..c2ec74b99 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,13 +70,23 @@ jobs: - uses: astral-sh/setup-uv@v6 - # free some space to prevent reaching GHA disk space limits - - name: Clean docker images + - name: Free up disk space if: runner.os == 'Linux' run: | docker system prune -a -f + sudo rm -rf $ANDROID_HOME/ndk/{26,28}.* /opt/hostedtoolcache/CodeQL \ + /usr/local/lib/node_modules /usr/local/share/chromium \ + /usr/local/share/powershell df -h + # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ + - name: Enable KVM for Android emulator + if: runner.os == 'Linux' && runner.arch == 'X64' + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + # for oci_container unit tests - name: Set up QEMU if: runner.os == 'Linux' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0bdea9190..cc9411891 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,13 +26,16 @@ repos: - id: mypy name: mypy 3.11 on cibuildwheel/ args: ["--python-version=3.11"] + exclude: ^cibuildwheel/resources/_cross_venv.py$ # Requires Python 3.13 or later additional_dependencies: &mypy-dependencies - bracex + - build - dependency-groups>=1.2 - humanize - nox>=2025.2.9 - orjson - packaging + - pyelftools - pygithub - pytest - rich @@ -47,7 +50,6 @@ repos: - validate-pyproject - id: mypy name: mypy 3.13 - exclude: ^cibuildwheel/resources/.*py$ args: ["--python-version=3.13"] additional_dependencies: *mypy-dependencies diff --git a/README.md b/README.md index 8c69f9ae0..6ac4156fc 100644 --- a/README.md +++ b/README.md @@ -23,20 +23,20 @@ What does it do? While cibuildwheel itself requires a recent Python version to run (we support the last three releases), it can target the following versions to build wheels: -| | macOS Intel | macOS Apple Silicon | Windows 64bit | Windows 32bit | Windows Arm64 | manylinux
musllinux x86_64 | manylinux
musllinux i686 | manylinux
musllinux aarch64 | manylinux
musllinux ppc64le | manylinux
musllinux s390x | manylinux
musllinux armv7l | iOS | Pyodide | -|--------------------|----|-----|-----|-----|-----|----|-----|----|-----|-----|---|-----|-----| -| CPython 3.8 | ✅ | ✅ | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | -| CPython 3.9 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | -| CPython 3.10 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | -| CPython 3.11 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | -| CPython 3.12 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | ✅⁴ | -| CPython 3.13³ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | ✅ | N/A | -| CPython 3.14³ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | ✅ | N/A | -| PyPy 3.8 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | -| PyPy 3.9 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | -| PyPy 3.10 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | -| PyPy 3.11 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | -| GraalPy 3.11 v24.2 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | N/A | ✅¹ | N/A | N/A | N/A | N/A | N/A | +| | macOS Intel | macOS Apple Silicon | Windows 64bit | Windows 32bit | Windows Arm64 | manylinux
musllinux x86_64 | manylinux
musllinux i686 | manylinux
musllinux aarch64 | manylinux
musllinux ppc64le | manylinux
musllinux s390x | manylinux
musllinux armv7l | Android | iOS | Pyodide | +|--------------------|----|-----|----|-----|-----|----|-----|----|-----|-----|---|-----|-----|-----| +| CPython 3.8 | ✅ | ✅ | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | N/A | +| CPython 3.9 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | N/A | +| CPython 3.10 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | N/A | +| CPython 3.11 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | N/A | +| CPython 3.12 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | ✅⁴ | +| CPython 3.13³ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | ✅ | ✅ | N/A | +| CPython 3.14³ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅⁵ | N/A | N/A | N/A | +| PyPy 3.8 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | N/A | +| PyPy 3.9 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | N/A | +| PyPy 3.10 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | N/A | +| PyPy 3.11 v7.3 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | N/A | N/A | N/A | N/A | +| GraalPy 3.11 v24.2 | ✅ | ✅ | ✅ | N/A | N/A | ✅¹ | N/A | ✅¹ | N/A | N/A | N/A | N/A | N/A | N/A | ¹ PyPy & GraalPy are only supported for manylinux wheels.
² Windows arm64 support is experimental.
@@ -56,19 +56,19 @@ Usage `cibuildwheel` runs inside a CI service. Supported platforms depend on which service you're using: -| | Linux | macOS | Windows | Linux ARM | macOS ARM | Windows ARM | iOS | -|-----------------|-------|-------|---------|-----------|-----------|-------------|-----| -| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅³ | -| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅³ | -| Travis CI | ✅ | | ✅ | ✅ | | | | -| CircleCI | ✅ | ✅ | | ✅ | ✅ | | ✅³ | -| Gitlab CI | ✅ | ✅ | ✅ | ✅¹ | ✅ | | ✅³ | -| Cirrus CI | ✅ | ✅ | ✅ | ✅ | ✅ | | | +| | Linux | macOS | Windows | Linux ARM | macOS ARM | Windows ARM | Android | iOS | +|-----------------|-------|-------|---------|-----------|-----------|-------------|---------|-----| +| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅⁴ | ✅³ | +| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅⁴ | ✅³ | +| Travis CI | ✅ | | ✅ | ✅ | | | ✅⁴ | | +| CircleCI | ✅ | ✅ | | ✅ | ✅ | | ✅⁴ | ✅³ | +| Gitlab CI | ✅ | ✅ | ✅ | ✅¹ | ✅ | | ✅⁴ | ✅³ | +| Cirrus CI | ✅ | ✅ | ✅ | ✅ | ✅ | | ✅⁴ | | ¹ [Requires emulation](https://cibuildwheel.pypa.io/en/stable/faq/#emulation), distributed separately. Other services may also support Linux ARM through emulation or third-party build hosts, but these are not tested in our CI.
² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.
-³ Requires a macOS runner; runs tests on the simulator for the runner's architecture. - +³ Requires a macOS runner; runs tests on the simulator for the runner's architecture.
+⁴ Building for Android requires the runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Testing has [additional requirements](https://cibuildwheel.pypa.io/en/stable/platforms/#android).
Example setup @@ -151,7 +151,7 @@ The following diagram summarises the steps that cibuildwheel takes on each platf | | [`pyodide-version`](https://cibuildwheel.pypa.io/en/stable/options/#pyodide-version) | Specify the Pyodide version to use for `pyodide` platform builds | | **Testing** | [`test-command`](https://cibuildwheel.pypa.io/en/stable/options/#test-command) | The command to test each built wheel | | | [`before-test`](https://cibuildwheel.pypa.io/en/stable/options/#before-test) | Execute a shell command before testing each wheel | -| | [`test-sources`](https://cibuildwheel.pypa.io/en/stable/options/#test-sources) | Files and folders from the source tree that are copied into an isolated tree before running the tests | +| | [`test-sources`](https://cibuildwheel.pypa.io/en/stable/options/#test-sources) | Paths that are copied into the working directory of the tests | | | [`test-requires`](https://cibuildwheel.pypa.io/en/stable/options/#test-requires) | Install Python dependencies before running the tests | | | [`test-extras`](https://cibuildwheel.pypa.io/en/stable/options/#test-extras) | Install your wheel for testing using `extras_require` | | | [`test-groups`](https://cibuildwheel.pypa.io/en/stable/options/#test-groups) | Specify test dependencies from your project's `dependency-groups` | @@ -162,7 +162,7 @@ The following diagram summarises the steps that cibuildwheel takes on each platf | | [`build-verbosity`](https://cibuildwheel.pypa.io/en/stable/options/#build-verbosity) | Increase/decrease the output of the build | - + These options can be specified in a pyproject.toml file, or as environment variables, see [configuration docs](https://cibuildwheel.pypa.io/en/latest/configuration/). @@ -245,10 +245,10 @@ See @henryiii's [release post](https://iscinumpy.dev/post/cibuildwheel-3-0-0/) f - ✨ Adds CPython 3.14 support, under the [`enable` option](https://cibuildwheel.pypa.io/en/stable/options/#enable) `cpython-prerelease`. This version of cibuildwheel uses 3.14.0b2. (#2390) _While CPython is in beta, the ABI can change, so your wheels might not be compatible with the final release. For this reason, we don't recommend distributing wheels until RC1, at which point 3.14 will be available in cibuildwheel without the flag._ (#2390) -- ✨ Adds the [test-sources option](https://cibuildwheel.pypa.io/en/stable/options/#test-sources), and changes the working directory for tests. (#2062, #2284, #2437) - - If this option is set, cibuildwheel will copy the files and folders specified in `test-sources` into the temporary directory we run from. This is required for iOS builds, but also useful for other platforms, as it allows you to avoid placeholders. - - If this option is not set, behaviour matches v2.x - cibuildwheel will run the tests from a temporary directory, and you can use the `{project}` placeholder in the `test-command` to refer to the project directory. (#2420) +- ✨ Adds the [test-sources option](https://cibuildwheel.pypa.io/en/stable/options/#test-sources), which copies files and folders into the temporary working directory we run tests from. (#2062, #2284, #2420, #2437) + + This is particularly important for iOS builds, which do not support placeholders in the `test-command`, but can also be useful for other platforms. - ✨ Adds [`dependency-versions`](https://cibuildwheel.pypa.io/en/stable/options/#dependency-versions) inline syntax (#2122) - ✨ Improves support for Pyodide builds and adds the experimental [`pyodide-version`](https://cibuildwheel.pypa.io/en/stable/options/#pyodide-version) option, which allows you to specify the version of Pyodide to use for builds. (#2002) @@ -292,7 +292,7 @@ _15 March 2025_ - ⚠️ Added warnings when the shorthand values `manylinux1`, `manylinux2010`, `manylinux_2_24`, and `musllinux_1_1` are used to specify the images in linux builds. The shorthand to these (unmaintainted) images will be removed in v3.0. If you want to keep using these images, explicitly opt-in using the full image URL, which can be found in [this file](https://github.com/pypa/cibuildwheel/blob/v2.23.1/cibuildwheel/resources/pinned_docker_images.cfg). (#2312) - 🛠 Dependency updates, including a manylinux update which fixes an [issue with rustup](https://github.com/pypa/cibuildwheel/issues/2303). (#2315) - + --- diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 23d6928e0..67dfac458 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -23,6 +23,11 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '3.11' + - task: JavaToolInstaller@0 + inputs: + versionSpec: '17' + jdkArchitectureOption: 'x64' + jdkSourceOption: 'PreInstalled' - bash: | docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all python -m pip install -U pip diff --git a/bin/generate_schema.py b/bin/generate_schema.py index cc4e8730f..df0bfdf01 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -348,6 +348,7 @@ def as_object(d: dict[str, Any]) -> dict[str, Any]: "windows": as_object(not_linux), "macos": as_object(not_linux), "pyodide": as_object(not_linux), + "android": as_object(not_linux), "ios": as_object(not_linux), } diff --git a/bin/update_pythons.py b/bin/update_pythons.py index d1058f452..20a99861d 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -10,6 +10,7 @@ from collections.abc import Mapping, MutableMapping from pathlib import Path from typing import Any, Final, Literal, TypedDict +from xml.etree import ElementTree as ET import click import requests @@ -20,6 +21,7 @@ from rich.syntax import Syntax from cibuildwheel.extra import dump_python_configurations, get_pyodide_xbuildenv_info +from cibuildwheel.platforms.android import android_triplet log = logging.getLogger("cibw") @@ -57,6 +59,12 @@ class ConfigApple(TypedDict): url: str +class ConfigAndroid(TypedDict): + identifier: str + version: str + url: str + + class ConfigPyodide(TypedDict): identifier: str version: str @@ -64,7 +72,7 @@ class ConfigPyodide(TypedDict): node_version: str -AnyConfig = ConfigWinCP | ConfigWinPP | ConfigWinGP | ConfigApple | ConfigPyodide +AnyConfig = ConfigWinCP | ConfigWinPP | ConfigWinGP | ConfigApple | ConfigAndroid | ConfigPyodide # The following set of "Versions" classes allow the initial call to the APIs to @@ -320,6 +328,39 @@ def update_version_macos( return None +class AndroidVersions: + # This should be replaced with official python.org downloads once they're available. + MAVEN_URL = "https://repo.maven.apache.org/maven2/com/chaquo/python/python" + + def __init__(self) -> None: + response = requests.get(f"{self.MAVEN_URL}/maven-metadata.xml") + response.raise_for_status() + root = ET.fromstring(response.text) + + self.versions: list[Version] = [] + for version_elem in root.findall("./versioning/versions/version"): + version_str = version_elem.text + assert isinstance(version_str, str), version_str + self.versions.append(Version(version_str)) + + def update_version_android( + self, identifier: str, version: Version, spec: Specifier + ) -> ConfigAndroid | None: + sorted_versions = sorted(spec.filter(self.versions), reverse=True) + + # Return a config using the highest version for the given specifier. + if sorted_versions: + max_version = sorted_versions[0] + triplet = android_triplet(identifier) + return ConfigAndroid( + identifier=identifier, + version=str(version), + url=f"{self.MAVEN_URL}/{max_version}/python-{max_version}-{triplet}.tar.gz", + ) + else: + return None + + class CPythonIOSVersions: def __init__(self) -> None: response = requests.get( @@ -413,6 +454,7 @@ def __init__(self) -> None: self.macos_pypy = PyPyVersions("64") self.macos_pypy_arm64 = PyPyVersions("ARM64") + self.android = AndroidVersions() self.ios_cpython = CPythonIOSVersions() self.graalpy = GraalPyVersions() @@ -455,6 +497,8 @@ def update_config(self, config: MutableMapping[str, str]) -> None: config_update = self.windows_t_arm64.update_version_windows(spec) elif "win_arm64" in identifier and identifier.startswith("cp"): config_update = self.windows_arm64.update_version_windows(spec) + elif "android" in identifier: + config_update = self.android.update_version_android(identifier, version, spec) elif "ios" in identifier: config_update = self.ios_cpython.update_version_ios(identifier, version) elif "pyodide" in identifier: @@ -490,17 +534,9 @@ def update_pythons(force: bool, level: str) -> None: with toml_file_path.open("rb") as f: configs = tomllib.load(f) - for config in configs["windows"]["python_configurations"]: - all_versions.update_config(config) - - for config in configs["macos"]["python_configurations"]: - all_versions.update_config(config) - - for config in configs["ios"]["python_configurations"]: - all_versions.update_config(config) - - for config in configs["pyodide"]["python_configurations"]: - all_versions.update_config(config) + for platform in ["windows", "macos", "android", "ios", "pyodide"]: + for config in configs[platform]["python_configurations"]: + all_versions.update_config(config) result_toml = dump_python_configurations(configs) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 7585ca830..366e80b1d 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -21,7 +21,7 @@ from cibuildwheel.ci import CIProvider, detect_ci_provider, fix_ansi_codes_for_github_actions from cibuildwheel.logger import log from cibuildwheel.options import CommandLineArguments, Options, compute_options -from cibuildwheel.platforms import ALL_PLATFORM_MODULES, get_build_identifiers +from cibuildwheel.platforms import ALL_PLATFORM_MODULES, get_build_identifiers, native_platform from cibuildwheel.selector import BuildSelector, EnableGroup, selector_matches from cibuildwheel.typing import PLATFORMS, PlatformName from cibuildwheel.util.file import CIBW_CACHE_PATH @@ -93,14 +93,14 @@ def main_inner(global_options: GlobalOptions) -> None: parser.add_argument( "--platform", - choices=["auto", "linux", "macos", "windows", "pyodide", "ios"], + choices=["auto", "linux", "macos", "windows", "pyodide", "android", "ios"], default=None, help=""" Platform to build for. Use this option to override the auto-detected platform. Specifying "macos" or "windows" only works on that operating system. "linux" works on any desktop OS, as long as - Docker/Podman is installed. "pyodide" only works on linux and macOS. - "ios" only work on macOS. Default: auto. + Docker/Podman is installed. "pyodide" and "android" only work on + Linux and macOS. "ios" only works on macOS. Default: auto. """, ) @@ -241,28 +241,14 @@ def _compute_platform_only(only: str) -> PlatformName: return "windows" if "pyodide_" in only: return "pyodide" + if "android_" in only: + return "android" if "ios_" in only: return "ios" msg = f"Invalid --only='{only}', must be a build selector with a known platform" raise errors.ConfigurationError(msg) -def _compute_platform_auto() -> PlatformName: - if sys.platform.startswith("linux"): - return "linux" - elif sys.platform == "darwin": - return "macos" - elif sys.platform == "win32": - return "windows" - else: - msg = ( - 'Unable to detect platform from "sys.platform". cibuildwheel doesn\'t ' - "support building wheels for this platform. You might be able to build for a different " - "platform using the --platform argument. Check --help output for more information." - ) - raise errors.ConfigurationError(msg) - - def _compute_platform(args: CommandLineArguments) -> PlatformName: platform_option_value = args.platform or os.environ.get("CIBW_PLATFORM", "") or "auto" @@ -282,7 +268,7 @@ def _compute_platform(args: CommandLineArguments) -> PlatformName: elif platform_option_value != "auto": return typing.cast(PlatformName, platform_option_value) - return _compute_platform_auto() + return native_platform() def build_in_directory(args: CommandLineArguments) -> None: diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index 69c164b27..bf0af268f 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -17,16 +17,25 @@ "macos": "macOS", "windows": "Windows", "pyodide": "Pyodide", + "android": "Android", "ios": "iOS", } ARCH_SYNONYMS: Final[list[dict[PlatformName, str | None]]] = [ - {"linux": "x86_64", "macos": "x86_64", "windows": "AMD64"}, + {"linux": "x86_64", "macos": "x86_64", "windows": "AMD64", "android": "x86_64"}, {"linux": "i686", "macos": None, "windows": "x86"}, - {"linux": "aarch64", "macos": "arm64", "windows": "ARM64"}, + {"linux": "aarch64", "macos": "arm64", "windows": "ARM64", "android": "arm64_v8a"}, ] +def arch_synonym(arch: str, from_platform: PlatformName, to_platform: PlatformName) -> str | None: + for arch_synonym_ in ARCH_SYNONYMS: + if arch == arch_synonym_.get(from_platform): + return arch_synonym_.get(to_platform, arch) + + return arch + + def _check_aarch32_el0() -> bool: """Check if running armv7l natively on aarch64 is supported""" if not sys.platform.startswith("linux"): @@ -42,7 +51,7 @@ def _check_aarch32_el0() -> bool: @typing.final class Architecture(StrEnum): - # mac/linux archs + # mac/linux/android archs x86_64 = auto() # linux archs @@ -65,6 +74,9 @@ class Architecture(StrEnum): # WebAssembly wasm32 = auto() + # android archs + arm64_v8a = auto() + # iOS "multiarch" architectures that include both # the CPU architecture and the ABI. arm64_iphoneos = auto() @@ -123,15 +135,12 @@ def native_arch(platform: PlatformName) -> "Architecture | None": # we might need to rename the native arch to the machine we're running # on, as the same arch can have different names on different platforms if host_platform != platform: - for arch_synonym in ARCH_SYNONYMS: - if native_machine == arch_synonym.get(host_platform): - synonym = arch_synonym[platform] - - if synonym is None: - # can't build anything on this platform - return None + synonym = arch_synonym(native_machine, host_platform, platform) + if synonym is None: + # can't build anything on this platform + return None - native_architecture = Architecture(synonym) + native_architecture = Architecture(synonym) return native_architecture @@ -166,6 +175,7 @@ def all_archs(platform: PlatformName) -> "set[Architecture]": "macos": {Architecture.x86_64, Architecture.arm64, Architecture.universal2}, "windows": {Architecture.x86, Architecture.AMD64, Architecture.ARM64}, "pyodide": {Architecture.wasm32}, + "android": {Architecture.x86_64, Architecture.arm64_v8a}, "ios": { Architecture.x86_64_iphonesimulator, Architecture.arm64_iphonesimulator, diff --git a/cibuildwheel/frontend.py b/cibuildwheel/frontend.py index 18157cdd0..7973354fe 100644 --- a/cibuildwheel/frontend.py +++ b/cibuildwheel/frontend.py @@ -58,6 +58,22 @@ def _split_config_settings(config_settings: str) -> list[str]: return [f"-C{setting}" for setting in config_settings_list] +# Based on build.__main__.main. +def parse_config_settings(config_settings_str: str) -> dict[str, str | list[str]]: + config_settings: dict[str, str | list[str]] = {} + for arg in shlex.split(config_settings_str): + setting, _, value = arg.partition("=") + existing_value = config_settings.get(setting) + if existing_value is None: + config_settings[setting] = value + elif isinstance(existing_value, str): + config_settings[setting] = [existing_value, value] + else: + existing_value.append(value) + + return config_settings + + def get_build_frontend_extra_flags( build_frontend: BuildFrontendConfig, verbosity_level: int, config_settings: str ) -> list[str]: diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 6ed3b761c..c2b90775d 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -50,6 +50,8 @@ "macosx_universal2": "macOS Universal 2 - x86_64 and arm64", "macosx_arm64": "macOS arm64 - Apple Silicon", "pyodide_wasm32": "Pyodide", + "android_arm64_v8a": "Android arm64_v8a", + "android_x86_64": "Android x86_64", "ios_arm64_iphoneos": "iOS Device (ARM64)", "ios_arm64_iphonesimulator": "iOS Simulator (ARM64)", "ios_x86_64_iphonesimulator": "iOS Simulator (x86_64)", diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 6656df008..00deb0f8c 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -109,7 +109,7 @@ class BuildOptions: test_groups: list[str] test_environment: ParsedEnvironment build_verbosity: int - build_frontend: BuildFrontendConfig | None + build_frontend: BuildFrontendConfig config_settings: str container_engine: OCIContainerEngineConfig pyodide_version: str | None @@ -772,9 +772,8 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: env_plat=False, option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False), ) - build_frontend: BuildFrontendConfig | None if not build_frontend_str or build_frontend_str == "default": - build_frontend = None + build_frontend = BuildFrontendConfig("build") else: try: build_frontend = BuildFrontendConfig.from_config_string(build_frontend_str) diff --git a/cibuildwheel/platforms/__init__.py b/cibuildwheel/platforms/__init__.py index a56fb9036..63c4c37d2 100644 --- a/cibuildwheel/platforms/__init__.py +++ b/cibuildwheel/platforms/__init__.py @@ -1,12 +1,14 @@ from __future__ import annotations +import sys from collections.abc import Sequence from pathlib import Path from typing import Final, Protocol +from cibuildwheel import errors from cibuildwheel.architecture import Architecture from cibuildwheel.options import Options -from cibuildwheel.platforms import ios, linux, macos, pyodide, windows +from cibuildwheel.platforms import android, ios, linux, macos, pyodide, windows from cibuildwheel.selector import BuildSelector from cibuildwheel.typing import GenericPythonConfiguration, PlatformName @@ -28,10 +30,27 @@ def build(self, options: Options, tmp_path: Path) -> None: ... "windows": windows, "macos": macos, "pyodide": pyodide, + "android": android, "ios": ios, } +def native_platform() -> PlatformName: + if sys.platform.startswith("linux"): + return "linux" + elif sys.platform == "darwin": + return "macos" + elif sys.platform == "win32": + return "windows" + else: + msg = ( + 'Unable to detect platform from "sys.platform". cibuildwheel doesn\'t ' + "support building wheels for this platform. You might be able to build for a different " + "platform using the --platform argument. Check --help output for more information." + ) + raise errors.ConfigurationError(msg) + + def get_build_identifiers( platform_module: PlatformModule, build_selector: BuildSelector, diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py new file mode 100644 index 000000000..52d4898cb --- /dev/null +++ b/cibuildwheel/platforms/android.py @@ -0,0 +1,644 @@ +import csv +import hashlib +import os +import platform +import re +import shlex +import shutil +import subprocess +from collections.abc import Iterable, Iterator +from dataclasses import dataclass +from os.path import relpath +from pathlib import Path +from pprint import pprint +from runpy import run_path +from textwrap import dedent +from typing import Any + +from build import ProjectBuilder +from build.env import IsolatedEnv +from elftools.common.exceptions import ELFError +from elftools.elf.elffile import ELFFile +from filelock import FileLock + +from .. import errors, platforms +from ..architecture import Architecture, arch_synonym +from ..frontend import get_build_frontend_extra_flags, parse_config_settings +from ..logger import log +from ..options import BuildOptions, Options +from ..selector import BuildSelector +from ..util import resources +from ..util.cmd import call, shell +from ..util.file import CIBW_CACHE_PATH, copy_test_sources, download, move_file +from ..util.helpers import prepare_command +from ..util.packaging import find_compatible_wheel +from ..util.python_build_standalone import create_python_build_standalone_environment +from ..venv import constraint_flags, virtualenv + + +def android_triplet(identifier: str) -> str: + return { + "arm64_v8a": "aarch64-linux-android", + "x86_64": "x86_64-linux-android", + }[parse_identifier(identifier)[1]] + + +def parse_identifier(identifier: str) -> tuple[str, str]: + match = re.fullmatch(r"cp(\d)(\d+)-android_(.+)", identifier) + if not match: + msg = f"invalid Android identifier: '{identifier}'" + raise ValueError(msg) + major, minor, arch = match.groups() + return (f"{major}.{minor}", arch) + + +@dataclass(frozen=True) +class PythonConfiguration: + version: str + identifier: str + url: str + + @property + def arch(self) -> str: + return parse_identifier(self.identifier)[1] + + +def all_python_configurations() -> list[PythonConfiguration]: + return [PythonConfiguration(**item) for item in resources.read_python_configs("android")] + + +def get_python_configurations( + build_selector: BuildSelector, architectures: set[Architecture] +) -> list[PythonConfiguration]: + return [ + c + for c in all_python_configurations() + if c.arch in architectures and build_selector(c.identifier) + ] + + +def shell_prepared(command: str, *, build_options: BuildOptions, env: dict[str, str]) -> None: + shell( + prepare_command(command, project=".", package=build_options.package_dir), + env=env, + ) + + +def before_all(options: Options, python_configurations: list[PythonConfiguration]) -> None: + before_all_options = options.build_options(python_configurations[0].identifier) + if before_all_options.before_all: + log.step("Running before_all...") + shell_prepared( + before_all_options.before_all, + build_options=before_all_options, + env=before_all_options.environment.as_dictionary(os.environ), + ) + + +@dataclass(frozen=True) +class BuildState: + config: PythonConfiguration + options: BuildOptions + build_path: Path + python_dir: Path + build_env: dict[str, str] + android_env: dict[str, str] + + +def build(options: Options, tmp_path: Path) -> None: + if "ANDROID_HOME" not in os.environ: + msg = ( + "ANDROID_HOME environment variable is not set. For instructions, see " + "https://cibuildwheel.pypa.io/en/stable/platforms/#android" + ) + raise errors.FatalError(msg) + + configs = get_python_configurations( + options.globals.build_selector, options.globals.architectures + ) + if not configs: + return + + try: + before_all(options, configs) + + built_wheels: list[Path] = [] + for config in configs: + log.build_start(config.identifier) + build_options = options.build_options(config.identifier) + build_path = tmp_path / config.identifier + build_path.mkdir() + python_dir = setup_target_python(config, build_path) + build_env, android_env = setup_env(config, build_options, build_path, python_dir) + state = BuildState( + config, build_options, build_path, python_dir, build_env, android_env + ) + + compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) + if compatible_wheel: + print( + f"\nFound previously built wheel {compatible_wheel.name} that is " + f"compatible with {config.identifier}. Skipping build step..." + ) + repaired_wheel = compatible_wheel + else: + before_build(state) + built_wheel = build_wheel(state) + repaired_wheel = repair_wheel(state, built_wheel) + + test_wheel(state, repaired_wheel) + + output_wheel: Path | None = None + if compatible_wheel is None: + output_wheel = move_file( + repaired_wheel, build_options.output_dir / repaired_wheel.name + ) + built_wheels.append(output_wheel) + + shutil.rmtree(build_path) + log.build_end(output_wheel) + + except subprocess.CalledProcessError as error: + msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" + raise errors.FatalError(msg) from error + + +def setup_target_python(config: PythonConfiguration, build_path: Path) -> Path: + log.step("Installing target Python...") + python_tgz = CIBW_CACHE_PATH / config.url.rpartition("/")[-1] + with FileLock(f"{python_tgz}.lock"): + if not python_tgz.exists(): + download(config.url, python_tgz) + + python_dir = build_path / "python" + python_dir.mkdir() + shutil.unpack_archive(python_tgz, python_dir) + return python_dir + + +def setup_env( + config: PythonConfiguration, build_options: BuildOptions, build_path: Path, python_dir: Path +) -> tuple[dict[str, str], dict[str, str]]: + """ + Returns two environment dicts, both pointing at the same virtual environment: + + * build_env, which uses the environment normally. + * android_env, which uses the environment while simulating running on Android. + """ + log.step("Setting up build environment...") + + # Create virtual environment + python_exe = create_python_build_standalone_environment( + config.version, build_path, CIBW_CACHE_PATH + ) + venv_dir = build_path / "venv" + dependency_constraint = build_options.dependency_constraints.get_for_python_version( + version=config.version, tmp_dir=build_path + ) + build_env = virtualenv( + config.version, python_exe, venv_dir, dependency_constraint, use_uv=False + ) + create_cmake_toolchain(config, build_path, python_dir, build_env) + + # Apply custom environment variables, and check environment is still valid + build_env = build_options.environment.as_dictionary(build_env) + build_env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + for command in ["python", "pip"]: + which = call("which", command, env=build_env, capture_stdout=True).strip() + if which != f"{venv_dir}/bin/{command}": + msg = ( + f"{command} available on PATH doesn't match our installed instance. If you " + f"have modified PATH, ensure that you don't overwrite cibuildwheel's entry " + f"or insert {command} above it." + ) + raise errors.FatalError(msg) + call(command, "--version", env=build_env) + + # Construct an altered environment which simulates running on Android. + android_env = setup_android_env(config, python_dir, venv_dir, build_env) + + # Install build tools + build_frontend = build_options.build_frontend + if build_frontend.name != "build": + msg = "Android requires the build frontend to be 'build'" + raise errors.FatalError(msg) + call("pip", "install", "build", *constraint_flags(dependency_constraint), env=build_env) + + # Build-time requirements must be queried within android_env, because + # `get_requires_for_build` can run arbitrary code in setup.py scripts, which may be + # affected by the target platform. However, the requirements must be installed + # within build_env, because they're going to run on the build machine. + # + # The `build` CLI doesn't support this combination, so we use its API to query the + # requirements, and then install them ourselves with pip. We'll later run `build` in + # the same environment, passing the `--no-isolation` option. + class AndroidEnv(IsolatedEnv): + @property + def python_executable(self) -> str: + return f"{venv_dir}/bin/python" + + def make_extra_environ(self) -> dict[str, str]: + return android_env + + pb = ProjectBuilder.from_isolated_env(AndroidEnv(), build_options.package_dir) + if pb.build_system_requires: + call("pip", "install", *pb.build_system_requires, env=build_env) + + requires_for_build = pb.get_requires_for_build( + "wheel", parse_config_settings(build_options.config_settings) + ) + if requires_for_build: + call("pip", "install", *requires_for_build, env=build_env) + + return build_env, android_env + + +def create_cmake_toolchain( + config: PythonConfiguration, build_path: Path, python_dir: Path, build_env: dict[str, str] +) -> None: + toolchain_path = build_path / "toolchain.cmake" + build_env["CMAKE_TOOLCHAIN_FILE"] = str(toolchain_path) + with open(toolchain_path, "w", encoding="UTF-8") as toolchain_file: + print( + dedent( + f"""\ + # To support as many build systems as possible, we use environment + # variables as the single source of truth for compiler flags and paths, + # so they don't need to be specified here. + + set(CMAKE_SYSTEM_NAME Android) + set(CMAKE_SYSTEM_PROCESSOR {android_triplet(config.identifier).split("-")[0]}) + + # Inhibit all of CMake's own NDK handling code. + set(CMAKE_SYSTEM_VERSION 1) + + # Tell CMake where to look for headers and libraries. + list(INSERT CMAKE_FIND_ROOT_PATH 0 {python_dir}/prefix) + set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) + set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) + """ + ), + file=toolchain_file, + ) + + +def localize_sysconfigdata( + python_dir: Path, build_env: dict[str, str], sysconfigdata_path: Path +) -> dict[str, Any]: + sysconfigdata: dict[str, Any] = run_path(str(sysconfigdata_path))["build_time_vars"] + with sysconfigdata_path.open("w", encoding="UTF-8") as f: + f.write("# Generated by cibuildwheel\n") + f.write("build_time_vars = ") + sysconfigdata = localized_vars(build_env, sysconfigdata, python_dir / "prefix") + pprint(sysconfigdata, stream=f, compact=True) + return sysconfigdata + + +def localized_vars( + build_env: dict[str, str], orig_vars: dict[str, Any], prefix: Path +) -> dict[str, Any]: + orig_prefix = orig_vars["prefix"] + localized_vars_ = {} + for key, value in orig_vars.items(): + # The host's sysconfigdata will include references to build-time paths. + # Update these to refer to the current prefix. + final = value + if isinstance(final, str): + final = final.replace(orig_prefix, str(prefix)) + + if key == "ANDROID_API_LEVEL": + if api_level := build_env.get(key): + final = int(api_level) + + # Build systems vary in whether FLAGS variables are read from sysconfig, and if so, + # whether they're replaced by environment variables or combined with them. Even + # setuptools has changed its behavior here + # (https://github.com/pypa/setuptools/issues/4836). + # + # Ensure consistency by clearing the sysconfig variables and letting the environment + # variables take effect alone. This will also work for any non-Python build systems + # which the build script may call. + elif key in ["CFLAGS", "CXXFLAGS", "LDFLAGS"]: + final = "" + + # These variables contain an embedded copy of LDFLAGS. + elif key in ["LDSHARED", "LDCXXSHARED"]: + final = final.removesuffix(" " + orig_vars["LDFLAGS"]) + + localized_vars_[key] = final + + return localized_vars_ + + +def setup_android_env( + config: PythonConfiguration, python_dir: Path, venv_dir: Path, build_env: dict[str, str] +) -> dict[str, str]: + site_packages = next(venv_dir.glob("lib/python*/site-packages")) + for suffix in ["pth", "py"]: + shutil.copy(resources.PATH / f"_cross_venv.{suffix}", site_packages) + + sysconfigdata_path = Path( + shutil.copy( + next(python_dir.glob("prefix/lib/python*/_sysconfigdata_*.py")), + site_packages, + ) + ) + sysconfigdata = localize_sysconfigdata(python_dir, build_env, sysconfigdata_path) + + # Activate the code in _cross_venv.py. + android_env = build_env.copy() + android_env["CIBW_HOST_TRIPLET"] = android_triplet(config.identifier) + + # Get the environment variables needed to build for Android (CC, CFLAGS, etc). These are + # generated by https://github.com/python/cpython/blob/main/Android/android-env.sh. + env_output = call(python_dir / "android.py", "env", env=build_env, capture_stdout=True) + + # shlex.split should produce a sequence alternating between: + # * the word "export" + # * a key=value string, without quotes + for i, token in enumerate(shlex.split(env_output)): + if i % 2 == 0: + assert token == "export", token + else: + key, sep, value = token.partition("=") + assert sep == "=", token + android_env[key] = value + + # localized_vars cleared the CFLAGS and CXXFLAGS in the sysconfigdata, but most + # packages take their optimization flags from these variables. Pass these flags via + # environment variables instead. + # + # We don't enable debug information, because it significantly increases binary size, + # and most Android app developers don't have the NDK installed, so they would have no + # way to strip it. + opt = " ".join(word for word in sysconfigdata["OPT"].split() if not word.startswith("-g")) + for key in ["CFLAGS", "CXXFLAGS"]: + android_env[key] += " " + opt + + # Format the environment so it can be pasted into a shell when debugging. + for key, value in sorted(android_env.items()): + if os.environ.get(key) != value: + print(f"export {key}={shlex.quote(value)}") + + return android_env + + +def before_build(state: BuildState) -> None: + if state.options.before_build: + log.step("Running before_build...") + shell_prepared( + state.options.before_build, + build_options=state.options, + env=state.build_env, + ) + + +def build_wheel(state: BuildState) -> Path: + log.step("Building wheel...") + built_wheel_dir = state.build_path / "built_wheel" + call( + "python", + "-m", + "build", + state.options.package_dir, + "--wheel", + "--no-isolation", + "--skip-dependency-check", + f"--outdir={built_wheel_dir}", + *get_build_frontend_extra_flags( + state.options.build_frontend, + state.options.build_verbosity, + state.options.config_settings, + ), + env=state.android_env, + ) + + built_wheels = list(built_wheel_dir.glob("*.whl")) + if len(built_wheels) != 1: + msg = f"{built_wheel_dir} contains {len(built_wheels)} wheels; expected 1" + raise errors.FatalError(msg) + built_wheel = built_wheels[0] + + if built_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + return built_wheel + + +def repair_wheel(state: BuildState, built_wheel: Path) -> Path: + log.step("Repairing wheel...") + repaired_wheel_dir = state.build_path / "repaired_wheel" + repaired_wheel_dir.mkdir() + + if state.options.repair_command: + shell( + prepare_command( + state.options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir + ), + env=state.build_env, + ) + else: + repair_default(state.android_env, built_wheel, repaired_wheel_dir) + + repaired_wheels = list(repaired_wheel_dir.glob("*.whl")) + if len(repaired_wheels) == 0: + raise errors.RepairStepProducedNoWheelError() + if len(repaired_wheels) != 1: + msg = f"{repaired_wheel_dir} contains {len(repaired_wheels)} wheels; expected 1" + raise errors.FatalError(msg) + repaired_wheel = repaired_wheels[0] + + if repaired_wheel.name.endswith("none-any.whl"): + raise errors.NonPlatformWheelError() + return repaired_wheel + + +def repair_default( + android_env: dict[str, str], built_wheel: Path, repaired_wheel_dir: Path +) -> None: + """ + Adds libc++ to the wheel if anything links against it. In the future this should be + moved to auditwheel and generalized to support more libraries. + """ + if (match := re.search(r"^(.+?)-", built_wheel.name)) is None: + msg = f"Failed to parse wheel filename: {built_wheel.name}" + raise errors.FatalError(msg) + wheel_name = match[1] + + unpacked_dir = repaired_wheel_dir / "unpacked" + unpacked_dir.mkdir() + shutil.unpack_archive(built_wheel, unpacked_dir, format="zip") + + # Some build systems are inconsistent about name normalization, so don't assume the + # dist-info name is identical to the wheel name. + record_paths = list(unpacked_dir.glob("*.dist-info/RECORD")) + if len(record_paths) != 1: + msg = f"{built_wheel.name} contains {len(record_paths)} dist-info/RECORD files; expected 1" + raise errors.FatalError(msg) + + old_soname = "libc++_shared.so" + paths_to_patch = [] + for path, elffile in elf_file_filter( + unpacked_dir / filename + for filename, *_ in csv.reader(record_paths[0].read_text().splitlines()) + ): + if (dynamic := elffile.get_section_by_name(".dynamic")) and any( # type: ignore[no-untyped-call] + tag.entry.d_tag == "DT_NEEDED" and tag.needed == old_soname + for tag in dynamic.iter_tags() + ): + paths_to_patch.append(path) + + if not paths_to_patch: + shutil.copyfile(built_wheel, repaired_wheel_dir / built_wheel.name) + else: + # Android doesn't support DT_RPATH, but supports DT_RUNPATH since API level 24 + # (https://github.com/aosp-mirror/platform_bionic/blob/master/android-changes-for-ndk-developers.md). + if int(sysconfig_print('get_config_vars()["ANDROID_API_LEVEL"]', android_env)) < 24: + msg = f"Adding {old_soname} requires ANDROID_API_LEVEL to be at least 24" + raise errors.FatalError(msg) + + toolchain = Path(android_env["CC"]).parent.parent + src_path = toolchain / f"sysroot/usr/lib/{android_env['CIBW_HOST_TRIPLET']}/{old_soname}" + + # Use the same library location as auditwheel would. + libs_dir = unpacked_dir / (wheel_name + ".libs") + libs_dir.mkdir() + new_soname = soname_with_hash(src_path) + dst_path = libs_dir / new_soname + shutil.copyfile(src_path, dst_path) + call("patchelf", "--set-soname", new_soname, dst_path) + + for path in paths_to_patch: + call("patchelf", "--replace-needed", old_soname, new_soname, path) + call( + "patchelf", + "--set-rpath", + f"${{ORIGIN}}/{relpath(libs_dir, path.parent)}", + path, + ) + call("wheel", "pack", unpacked_dir, "-d", repaired_wheel_dir) + + +def elf_file_filter(paths: Iterable[Path]) -> Iterator[tuple[Path, ELFFile]]: + """Filter through an iterator of filenames and load up only ELF files""" + for path in paths: + if not path.name.endswith(".py"): + try: + with open(path, "rb") as f: + candidate = ELFFile(f) # type: ignore[no-untyped-call] + yield path, candidate + except ELFError: + pass # Not an ELF file + + +def soname_with_hash(src_path: Path) -> str: + """Return the same library filename as auditwheel would""" + shorthash = hashlib.sha256(src_path.read_bytes()).hexdigest()[:8] + src_name = src_path.name + base, ext = src_name.split(".", 1) + if not base.endswith(f"-{shorthash}"): + return f"{base}-{shorthash}.{ext}" + else: + return src_name + + +def test_wheel(state: BuildState, wheel: Path) -> None: + test_command = state.options.test_command + if not (test_command and state.options.test_selector(state.config.identifier)): + return + + log.step("Testing wheel...") + native_arch = arch_synonym(platform.machine(), platforms.native_platform(), "android") + if state.config.arch != native_arch: + log.warning( + f"Skipping tests for {state.config.arch}, as the build machine only " + f"supports {native_arch}" + ) + return + + if state.options.before_test: + shell_prepared( + state.options.before_test, + build_options=state.options, + env=state.build_env, + ) + + # Install the wheel and test-requires. + site_packages_dir = state.build_path / "site-packages" + site_packages_dir.mkdir() + call( + "pip", + "install", + "--only-binary=:all:", + "--platform", + sysconfig_print("get_platform()", state.android_env).replace("-", "_"), + "--target", + site_packages_dir, + f"{wheel}{state.options.test_extras}", + *state.options.test_requires, + env=state.build_env, + ) + + # Copy test-sources. + cwd_dir = state.build_path / "cwd" + cwd_dir.mkdir() + if state.options.test_sources: + copy_test_sources(state.options.test_sources, Path.cwd(), cwd_dir) + else: + (cwd_dir / "test_fail.py").write_text( + resources.TEST_FAIL_CWD_FILE.read_text(), + ) + + # Android doesn't support placeholders in the test command. + if any(("{" + placeholder + "}") in test_command for placeholder in ["project", "package"]): + msg = ( + f"Test command {test_command!r} with a " + "'{project}' or '{package}' placeholder is not supported on Android, " + "because the source directory is not visible on the emulator." + ) + raise errors.FatalError(msg) + + # Parse test-command. + test_args = shlex.split(test_command) + if test_args[:2] in [["python", "-c"], ["python", "-m"]]: + test_args[:3] = [test_args[1], test_args[2], "--"] + elif test_args[0] in ["pytest"]: + # We transform some commands into the `python -m` form, but this is deprecated. + msg = ( + f"Test command {test_command!r} is not supported on Android. " + "cibuildwheel will try to execute it as if it started with 'python -m'. " + "If this works, all you need to do is add that to your test command." + ) + log.warning(msg) + test_args[:1] = ["-m", test_args[0], "--"] + else: + msg = ( + f"Test command {test_command!r} is not supported on Android. " + f"Supported commands are 'python -m' and 'python -c'." + ) + raise errors.FatalError(msg) + + # Run the test app. + call( + state.python_dir / "android.py", + "test", + "--managed", + "maxVersion", + "--site-packages", + site_packages_dir, + "--cwd", + cwd_dir, + *test_args, + env=state.build_env, + ) + + +def sysconfig_print(method_call: str, env: dict[str, str]) -> str: + return call( + "python", + "-c", + f'import sysconfig; print(sysconfig.{method_call}, end="")', + env=env, + capture_stdout=True, + ) diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index f4cf6b60d..eba04c35d 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -17,7 +17,6 @@ from ..architecture import Architecture from ..environment import ParsedEnvironment from ..frontend import ( - BuildFrontendConfig, BuildFrontendName, get_build_frontend_extra_flags, ) @@ -438,7 +437,7 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + build_frontend = build_options.build_frontend # uv doesn't support iOS if build_frontend.name == "build[uv]": msg = "uv doesn't support iOS" diff --git a/cibuildwheel/platforms/linux.py b/cibuildwheel/platforms/linux.py index 8d25d73d0..8411a935a 100644 --- a/cibuildwheel/platforms/linux.py +++ b/cibuildwheel/platforms/linux.py @@ -10,7 +10,7 @@ from .. import errors from ..architecture import Architecture -from ..frontend import BuildFrontendConfig, get_build_frontend_extra_flags +from ..frontend import get_build_frontend_extra_flags from ..logger import log from ..oci_container import OCIContainer, OCIContainerEngineConfig, OCIPlatform from ..options import BuildOptions, Options @@ -204,7 +204,7 @@ def build_in_container( log.build_start(config.identifier) local_identifier_tmp_dir = local_tmp_dir / config.identifier build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + build_frontend = build_options.build_frontend use_uv = build_frontend.name == "build[uv]" pip = ["uv", "pip"] if use_uv else ["pip"] diff --git a/cibuildwheel/platforms/macos.py b/cibuildwheel/platforms/macos.py index a273f062c..05d5d1623 100644 --- a/cibuildwheel/platforms/macos.py +++ b/cibuildwheel/platforms/macos.py @@ -19,7 +19,7 @@ from ..architecture import Architecture from ..ci import detect_ci_provider from ..environment import ParsedEnvironment -from ..frontend import BuildFrontendConfig, BuildFrontendName, get_build_frontend_extra_flags +from ..frontend import BuildFrontendName, get_build_frontend_extra_flags from ..logger import log from ..options import Options from ..selector import BuildSelector @@ -408,7 +408,7 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + build_frontend = build_options.build_frontend use_uv = build_frontend.name == "build[uv]" uv_path = find_uv() if use_uv and uv_path is None: diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index c46248ee7..1871fca40 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -17,7 +17,7 @@ from .. import errors from ..architecture import Architecture from ..environment import ParsedEnvironment -from ..frontend import BuildFrontendConfig, get_build_frontend_extra_flags +from ..frontend import get_build_frontend_extra_flags from ..logger import log from ..options import Options from ..selector import BuildSelector @@ -353,7 +353,7 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + build_frontend = build_options.build_frontend if build_frontend.name == "pip": msg = "The pyodide platform doesn't support pip frontend" diff --git a/cibuildwheel/platforms/windows.py b/cibuildwheel/platforms/windows.py index 5a55b3a08..f9c85be89 100644 --- a/cibuildwheel/platforms/windows.py +++ b/cibuildwheel/platforms/windows.py @@ -15,7 +15,7 @@ from .. import errors from ..architecture import Architecture from ..environment import ParsedEnvironment -from ..frontend import BuildFrontendConfig, BuildFrontendName, get_build_frontend_extra_flags +from ..frontend import BuildFrontendName, get_build_frontend_extra_flags from ..logger import log from ..options import Options from ..selector import BuildSelector @@ -395,7 +395,7 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) - build_frontend = build_options.build_frontend or BuildFrontendConfig("build") + build_frontend = build_options.build_frontend use_uv = build_frontend.name == "build[uv]" and can_use_uv(config) log.build_start(config.identifier) diff --git a/cibuildwheel/resources/_cross_venv.pth b/cibuildwheel/resources/_cross_venv.pth new file mode 100644 index 000000000..c8a04bc64 --- /dev/null +++ b/cibuildwheel/resources/_cross_venv.pth @@ -0,0 +1 @@ +import _cross_venv; _cross_venv.initialize() diff --git a/cibuildwheel/resources/_cross_venv.py b/cibuildwheel/resources/_cross_venv.py new file mode 100644 index 000000000..84237a2eb --- /dev/null +++ b/cibuildwheel/resources/_cross_venv.py @@ -0,0 +1,83 @@ +# This module is copied into the site-packages directory of an Android build environment, and +# activated via a .pth file when we want the environment to simulate Android. + +import os +import platform +import re +import sys +import sysconfig +from pathlib import Path +from typing import Any + + +def initialize() -> None: + if not (host_triplet := os.environ.get("CIBW_HOST_TRIPLET")): + return + + # os ###################################################################### + def cross_os_uname() -> os.uname_result: + return os.uname_result( + ( + "Linux", + "localhost", + # The Linux kernel version and release are unlikely to be significant, but return + # realistic values anyway (from an API level 24 emulator). + "3.18.91+", + "#1 SMP PREEMPT Tue Jan 9 20:35:43 UTC 2018", + host_triplet.split("-")[0], + ) + ) + + os.name = "posix" + os.uname = cross_os_uname + + # platform ################################################################ + # + # We can't determine the user-visible Android version number from the API level, so return a + # string which will work fine for display, but will fail to parse as a version number. + def cross_android_ver(*args: Any, **kwargs: Any) -> platform.AndroidVer: + return platform.AndroidVer( + release=f"API level {cross_getandroidapilevel()}", + api_level=cross_getandroidapilevel(), + manufacturer="Google", + model="sdk_gphone64", + device="emu64", + is_emulator=True, + ) + + # platform.uname, platform.system etc. are all implemented in terms of platform.android_ver. + platform.android_ver = cross_android_ver + + # sys ##################################################################### + def cross_getandroidapilevel() -> int: + api_level = sysconfig.get_config_var("ANDROID_API_LEVEL") + assert isinstance(api_level, int) + return api_level + + # Some packages may recognize sys.cross_compiling from the crossenv tool. + sys.cross_compiling = True # type: ignore[attr-defined] + sys.getandroidapilevel = cross_getandroidapilevel # type: ignore[attr-defined] + sys.implementation._multiarch = host_triplet # type: ignore[attr-defined] + sys.platform = "android" + + # Determine the abiflags from the sysconfigdata filename. + sysconfigdata_path = next(Path(__file__).parent.glob("_sysconfigdata_*.py")) + abiflags_match = re.match(r"_sysconfigdata_(.*?)_", sysconfigdata_path.name) + assert abiflags_match is not None + sys.abiflags = abiflags_match[1] + + # sysconfig ############################################################### + # + # We don't change the actual sys.base_prefix and base_exec_prefix, because that + # could have unpredictable effects. Instead, we change the internal variables + # used to generate sysconfig.get_path("include"). + exec_prefix = sysconfig.get_config_var("exec_prefix") + sysconfig._BASE_PREFIX = sysconfig._BASE_EXEC_PREFIX = exec_prefix # type: ignore[attr-defined] + + # Reload the sysconfigdata file, generating its name from sys.abiflags, + # sys.platform, and sys.implementation._multiarch. + sysconfig._init_config_vars() # type: ignore[attr-defined] + + # sysconfig.get_platform, which determines the wheel tag, is implemented in terms of + # sys.platform, sysconfig.get_config_var("ANDROID_API_LEVEL") (see localized_vars in + # android.py), and os.uname. diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index cbef21ca1..371ae0d2a 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -224,6 +224,12 @@ python_configurations = [ { identifier = "cp313-pyodide_wasm32", version = "3.13", default_pyodide_version = "0.28.0", node_version = "v22" }, ] +[android] +python_configurations = [ + { identifier = "cp313-android_arm64_v8a", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.5+20250722.214220/python-3.13.5+20250722.214220-aarch64-linux-android.tar.gz" }, + { identifier = "cp313-android_x86_64", version = "3.13", url = "https://repo.maven.apache.org/maven2/com/chaquo/python/python/3.13.5+20250722.214220/python-3.13.5+20250722.214220-x86_64-linux-android.tar.gz" }, +] + [ios] python_configurations = [ { identifier = "cp313-ios_arm64_iphoneos", version = "3.13", url = "https://github.com/beeware/Python-Apple-support/releases/download/3.13-b9/Python-3.13-iOS-support.b9.tar.gz" }, diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 7d6aaf10d..1b00884d3 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -108,7 +108,7 @@ }, "build-frontend": { "default": "default", - "description": "Set the tool to use to build, either \"pip\" (default for now), \"build\", or \"build[uv]\"", + "description": "Set the tool to use to build, either \"build\" (default), \"build[uv]\", or \"pip\"", "oneOf": [ { "enum": [ @@ -1072,6 +1072,57 @@ } } }, + "android": { + "type": "object", + "additionalProperties": false, + "properties": { + "archs": { + "$ref": "#/properties/archs" + }, + "before-all": { + "$ref": "#/properties/before-all" + }, + "before-build": { + "$ref": "#/properties/before-build" + }, + "before-test": { + "$ref": "#/properties/before-test" + }, + "build-frontend": { + "$ref": "#/properties/build-frontend" + }, + "build-verbosity": { + "$ref": "#/properties/build-verbosity" + }, + "config-settings": { + "$ref": "#/properties/config-settings" + }, + "dependency-versions": { + "$ref": "#/properties/dependency-versions" + }, + "environment": { + "$ref": "#/properties/environment" + }, + "repair-wheel-command": { + "$ref": "#/properties/repair-wheel-command" + }, + "test-command": { + "$ref": "#/properties/test-command" + }, + "test-extras": { + "$ref": "#/properties/test-extras" + }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, + "test-groups": { + "$ref": "#/properties/test-groups" + }, + "test-requires": { + "$ref": "#/properties/test-requires" + } + } + }, "ios": { "type": "object", "additionalProperties": false, diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 6a57f2d03..d05c0bf7e 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -58,6 +58,8 @@ repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest [tool.cibuildwheel.windows] +[tool.cibuildwheel.android] + [tool.cibuildwheel.ios] [tool.cibuildwheel.pyodide] diff --git a/cibuildwheel/resources/testing_temp_dir_file.py b/cibuildwheel/resources/testing_temp_dir_file.py index 094f2f6fb..cc3e15a2b 100644 --- a/cibuildwheel/resources/testing_temp_dir_file.py +++ b/cibuildwheel/resources/testing_temp_dir_file.py @@ -8,11 +8,11 @@ class TestStringMethods(unittest.TestCase): def test_fail(self) -> NoReturn: - if sys.platform == "ios": + if sys.platform in ["android", "ios"]: msg = ( "You tried to run tests from the testbed app's working " "directory, without specifying `test-sources`. " - "On iOS, you must copy your test files to the testbed app by " + "On this platform, you must copy your test files to the testbed app by " "setting the `test-sources` option in your cibuildwheel " "configuration." ) diff --git a/cibuildwheel/typing.py b/cibuildwheel/typing.py index 420ef05f6..53b0887da 100644 --- a/cibuildwheel/typing.py +++ b/cibuildwheel/typing.py @@ -12,7 +12,7 @@ PathOrStr = str | os.PathLike[str] -PlatformName = Literal["linux", "macos", "windows", "pyodide", "ios"] +PlatformName = Literal["linux", "macos", "windows", "pyodide", "android", "ios"] PLATFORMS: Final[frozenset[PlatformName]] = frozenset(typing.get_args(PlatformName)) diff --git a/cibuildwheel/util/packaging.py b/cibuildwheel/util/packaging.py index 295bc091c..578f82e44 100644 --- a/cibuildwheel/util/packaging.py +++ b/cibuildwheel/util/packaging.py @@ -158,9 +158,9 @@ def find_compatible_wheel(wheels: Sequence[T], identifier: str) -> T | None: # If a minor version number is given, it has to be lower than the current one. continue - if platform.startswith(("manylinux", "musllinux", "macosx", "ios")): - # Linux, macOS, and iOS require the beginning and ending match - # (macos/manylinux/iOS version number doesn't need to match) + if platform.startswith(("manylinux", "musllinux", "macosx", "android", "ios")): + # On these platforms the wheel tag includes a platform version number, which we + # should ignore. os_, arch = platform.split("_", 1) if not tag.platform.startswith(os_): continue diff --git a/docs/changelog.md b/docs/changelog.md index a6d61d49e..56eaf599e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -22,10 +22,10 @@ See @henryiii's [release post](https://iscinumpy.dev/post/cibuildwheel-3-0-0/) f - ✨ Adds CPython 3.14 support, under the [`enable` option](https://cibuildwheel.pypa.io/en/stable/options/#enable) `cpython-prerelease`. This version of cibuildwheel uses 3.14.0b2. (#2390) _While CPython is in beta, the ABI can change, so your wheels might not be compatible with the final release. For this reason, we don't recommend distributing wheels until RC1, at which point 3.14 will be available in cibuildwheel without the flag._ (#2390) -- ✨ Adds the [test-sources option](https://cibuildwheel.pypa.io/en/stable/options/#test-sources), and changes the working directory for tests. (#2062, #2284, #2437) - - If this option is set, cibuildwheel will copy the files and folders specified in `test-sources` into the temporary directory we run from. This is required for iOS builds, but also useful for other platforms, as it allows you to avoid placeholders. - - If this option is not set, behaviour matches v2.x - cibuildwheel will run the tests from a temporary directory, and you can use the `{project}` placeholder in the `test-command` to refer to the project directory. (#2420) +- ✨ Adds the [test-sources option](https://cibuildwheel.pypa.io/en/stable/options/#test-sources), which copies files and folders into the temporary working directory we run tests from. (#2062, #2284, #2420, #2437) + + This is particularly important for iOS builds, which do not support placeholders in the `test-command`, but can also be useful for other platforms. - ✨ Adds [`dependency-versions`](https://cibuildwheel.pypa.io/en/stable/options/#dependency-versions) inline syntax (#2122) - ✨ Improves support for Pyodide builds and adds the experimental [`pyodide-version`](https://cibuildwheel.pypa.io/en/stable/options/#pyodide-version) option, which allows you to specify the version of Pyodide to use for builds. (#2002) diff --git a/docs/ci-services.md b/docs/ci-services.md index 6564052f8..ecf4f567e 100644 --- a/docs/ci-services.md +++ b/docs/ci-services.md @@ -53,7 +53,7 @@ Commit this file, and push to GitHub - either to your default branch, or to a PR For more info on this file, check out the [docs](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions). -[`examples/github-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml) extends this minimal example to include iOS and Pyodide builds, and a demonstration of how to automatically upload the built wheels to PyPI. +[`examples/github-deploy.yml`](https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml) extends this minimal example to include Android, iOS and Pyodide builds, and a demonstration of how to automatically upload the built wheels to PyPI. ### Azure Pipelines [linux/mac/windows] {: #azure-pipelines} diff --git a/docs/options.md b/docs/options.md index d545adc19..0a98b44b3 100644 --- a/docs/options.md +++ b/docs/options.md @@ -8,7 +8,7 @@ > Override the auto-detected target platform -Options: `auto` `linux` `macos` `windows` `ios` `pyodide` +Options: `auto` `linux` `macos` `windows` `android` `ios` `pyodide` Default: `auto` @@ -16,6 +16,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. +- For `android` you need to be running on Linux or macOS, with an Android SDK installed. See [here](platforms.md#android) for more details. - For `ios` you need to be running on macOS, with Xcode and the iOS simulator installed. - For `pyodide`, you need a Linux or macOS machine. @@ -51,19 +52,19 @@ When both options are specified, both conditions are applied and only builds wit When setting the options, you can use shell-style globbing syntax, as per [fnmatch](https://docs.python.org/3/library/fnmatch.html) with the addition of curly bracket syntax `{option1,option2}`, provided by [bracex](https://pypi.org/project/bracex/). All the build identifiers supported by cibuildwheel are shown below:
-| | macOS | Windows | Linux Intel | Linux Other | iOS | pyodide (WASM) | -|---------------|------------------------------------------------------------------------|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|----------------------| -| Python 3.8 | cp38-macosx_x86_64
cp38-macosx_universal2
cp38-macosx_arm64 | cp38-win_amd64
cp38-win32 | cp38-manylinux_x86_64
cp38-manylinux_i686
cp38-musllinux_x86_64
cp38-musllinux_i686 | cp38-manylinux_aarch64
cp38-manylinux_ppc64le
cp38-manylinux_s390x
cp38-manylinux_armv7l
cp38-manylinux_riscv64
cp38-musllinux_aarch64
cp38-musllinux_ppc64le
cp38-musllinux_s390x
cp38-musllinux_armv7l
cp38-musllinux_riscv64 | | | -| Python 3.9 | cp39-macosx_x86_64
cp39-macosx_universal2
cp39-macosx_arm64 | cp39-win_amd64
cp39-win32
cp39-win_arm64 | cp39-manylinux_x86_64
cp39-manylinux_i686
cp39-musllinux_x86_64
cp39-musllinux_i686 | cp39-manylinux_aarch64
cp39-manylinux_ppc64le
cp39-manylinux_s390x
cp39-manylinux_armv7l
cp39-manylinux_riscv64
cp39-musllinux_aarch64
cp39-musllinux_ppc64le
cp39-musllinux_s390x
cp39-musllinux_armv7l
cp39-musllinux_riscv64 | | | -| Python 3.10 | cp310-macosx_x86_64
cp310-macosx_universal2
cp310-macosx_arm64 | cp310-win_amd64
cp310-win32
cp310-win_arm64 | cp310-manylinux_x86_64
cp310-manylinux_i686
cp310-musllinux_x86_64
cp310-musllinux_i686 | cp310-manylinux_aarch64
cp310-manylinux_ppc64le
cp310-manylinux_s390x
cp310-manylinux_armv7l
cp310-manylinux_riscv64
cp310-musllinux_aarch64
cp310-musllinux_ppc64le
cp310-musllinux_s390x
cp310-musllinux_armv7l
cp310-musllinux_riscv64 | | | -| Python 3.11 | cp311-macosx_x86_64
cp311-macosx_universal2
cp311-macosx_arm64 | cp311-win_amd64
cp311-win32
cp311-win_arm64 | cp311-manylinux_x86_64
cp311-manylinux_i686
cp311-musllinux_x86_64
cp311-musllinux_i686 | cp311-manylinux_aarch64
cp311-manylinux_ppc64le
cp311-manylinux_s390x
cp311-manylinux_armv7l
cp311-manylinux_riscv64
cp311-musllinux_aarch64
cp311-musllinux_ppc64le
cp311-musllinux_s390x
cp311-musllinux_armv7l
cp311-musllinux_riscv64 | | | -| Python 3.12 | cp312-macosx_x86_64
cp312-macosx_universal2
cp312-macosx_arm64 | cp312-win_amd64
cp312-win32
cp312-win_arm64 | cp312-manylinux_x86_64
cp312-manylinux_i686
cp312-musllinux_x86_64
cp312-musllinux_i686 | cp312-manylinux_aarch64
cp312-manylinux_ppc64le
cp312-manylinux_s390x
cp312-manylinux_armv7l
cp312-manylinux_riscv64
cp312-musllinux_aarch64
cp312-musllinux_ppc64le
cp312-musllinux_s390x
cp312-musllinux_armv7l
cp312-musllinux_riscv64 | | cp312-pyodide_wasm32 | -| Python 3.13 | cp313-macosx_x86_64
cp313-macosx_universal2
cp313-macosx_arm64 | cp313-win_amd64
cp313-win32
cp313-win_arm64 | cp313-manylinux_x86_64
cp313-manylinux_i686
cp313-musllinux_x86_64
cp313-musllinux_i686 | cp313-manylinux_aarch64
cp313-manylinux_ppc64le
cp313-manylinux_s390x
cp313-manylinux_armv7l
cp313-manylinux_riscv64
cp313-musllinux_aarch64
cp313-musllinux_ppc64le
cp313-musllinux_s390x
cp313-musllinux_armv7l
cp313-musllinux_riscv64 | cp313-ios_arm64_iphoneos
cp313-ios_arm64_iphonesimulator
cp313-ios_x86_64_iphonesimulator | cp313-pyodide_wasm32 | +| | macOS | Windows | Linux Intel | Linux Other | Android | iOS | pyodide (WASM) | +|---------------|------------------------------------------------------------------------|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------|---------------------------------------------------------------------------------------------------|----------------------| +| Python 3.8 | cp38-macosx_x86_64
cp38-macosx_universal2
cp38-macosx_arm64 | cp38-win_amd64
cp38-win32 | cp38-manylinux_x86_64
cp38-manylinux_i686
cp38-musllinux_x86_64
cp38-musllinux_i686 | cp38-manylinux_aarch64
cp38-manylinux_ppc64le
cp38-manylinux_s390x
cp38-manylinux_armv7l
cp38-manylinux_riscv64
cp38-musllinux_aarch64
cp38-musllinux_ppc64le
cp38-musllinux_s390x
cp38-musllinux_armv7l
cp38-musllinux_riscv64 | | | | +| Python 3.9 | cp39-macosx_x86_64
cp39-macosx_universal2
cp39-macosx_arm64 | cp39-win_amd64
cp39-win32
cp39-win_arm64 | cp39-manylinux_x86_64
cp39-manylinux_i686
cp39-musllinux_x86_64
cp39-musllinux_i686 | cp39-manylinux_aarch64
cp39-manylinux_ppc64le
cp39-manylinux_s390x
cp39-manylinux_armv7l
cp39-manylinux_riscv64
cp39-musllinux_aarch64
cp39-musllinux_ppc64le
cp39-musllinux_s390x
cp39-musllinux_armv7l
cp39-musllinux_riscv64 | | | | +| Python 3.10 | cp310-macosx_x86_64
cp310-macosx_universal2
cp310-macosx_arm64 | cp310-win_amd64
cp310-win32
cp310-win_arm64 | cp310-manylinux_x86_64
cp310-manylinux_i686
cp310-musllinux_x86_64
cp310-musllinux_i686 | cp310-manylinux_aarch64
cp310-manylinux_ppc64le
cp310-manylinux_s390x
cp310-manylinux_armv7l
cp310-manylinux_riscv64
cp310-musllinux_aarch64
cp310-musllinux_ppc64le
cp310-musllinux_s390x
cp310-musllinux_armv7l
cp310-musllinux_riscv64 | | | | +| Python 3.11 | cp311-macosx_x86_64
cp311-macosx_universal2
cp311-macosx_arm64 | cp311-win_amd64
cp311-win32
cp311-win_arm64 | cp311-manylinux_x86_64
cp311-manylinux_i686
cp311-musllinux_x86_64
cp311-musllinux_i686 | cp311-manylinux_aarch64
cp311-manylinux_ppc64le
cp311-manylinux_s390x
cp311-manylinux_armv7l
cp311-manylinux_riscv64
cp311-musllinux_aarch64
cp311-musllinux_ppc64le
cp311-musllinux_s390x
cp311-musllinux_armv7l
cp311-musllinux_riscv64 | | | | +| Python 3.12 | cp312-macosx_x86_64
cp312-macosx_universal2
cp312-macosx_arm64 | cp312-win_amd64
cp312-win32
cp312-win_arm64 | cp312-manylinux_x86_64
cp312-manylinux_i686
cp312-musllinux_x86_64
cp312-musllinux_i686 | cp312-manylinux_aarch64
cp312-manylinux_ppc64le
cp312-manylinux_s390x
cp312-manylinux_armv7l
cp312-manylinux_riscv64
cp312-musllinux_aarch64
cp312-musllinux_ppc64le
cp312-musllinux_s390x
cp312-musllinux_armv7l
cp312-musllinux_riscv64 | | | cp312-pyodide_wasm32 | +| Python 3.13 | cp313-macosx_x86_64
cp313-macosx_universal2
cp313-macosx_arm64 | cp313-win_amd64
cp313-win32
cp313-win_arm64 | cp313-manylinux_x86_64
cp313-manylinux_i686
cp313-musllinux_x86_64
cp313-musllinux_i686 | cp313-manylinux_aarch64
cp313-manylinux_ppc64le
cp313-manylinux_s390x
cp313-manylinux_armv7l
cp313-manylinux_riscv64
cp313-musllinux_aarch64
cp313-musllinux_ppc64le
cp313-musllinux_s390x
cp313-musllinux_armv7l
cp313-musllinux_riscv64 | cp313-android_arm64_v8a
cp313-android_x86_64 | cp313-ios_arm64_iphoneos
cp313-ios_arm64_iphonesimulator
cp313-ios_x86_64_iphonesimulator | cp313-pyodide_wasm32 | | Python 3.14 | cp314-macosx_x86_64
cp314-macosx_universal2
cp314-macosx_arm64 | cp314-win_amd64
cp314-win32
cp314-win_arm64 | cp314-manylinux_x86_64
cp314-manylinux_i686
cp314-musllinux_x86_64
cp314-musllinux_i686 | cp314-manylinux_aarch64
cp314-manylinux_ppc64le
cp314-manylinux_s390x
cp314-manylinux_armv7l
cp314-manylinux_riscv64
cp314-musllinux_aarch64
cp314-musllinux_ppc64le
cp314-musllinux_s390x
cp314-musllinux_armv7l
cp314-musllinux_riscv64 | cp314-ios_arm64_iphoneos
cp314-ios_arm64_iphonesimulator
cp314-ios_x86_64_iphonesimulator | | -| PyPy3.8 v7.3 | pp38-macosx_x86_64
pp38-macosx_arm64 | pp38-win_amd64 | pp38-manylinux_x86_64
pp38-manylinux_i686 | pp38-manylinux_aarch64 | | | -| PyPy3.9 v7.3 | pp39-macosx_x86_64
pp39-macosx_arm64 | pp39-win_amd64 | pp39-manylinux_x86_64
pp39-manylinux_i686 | pp39-manylinux_aarch64 | | | -| PyPy3.10 v7.3 | pp310-macosx_x86_64
pp310-macosx_arm64 | pp310-win_amd64 | pp310-manylinux_x86_64
pp310-manylinux_i686 | pp310-manylinux_aarch64 | | | -| PyPy3.11 v7.3 | pp311-macosx_x86_64
pp311-macosx_arm64 | pp311-win_amd64 | pp311-manylinux_x86_64
pp311-manylinux_i686 | pp311-manylinux_aarch64 | | | +| PyPy3.8 v7.3 | pp38-macosx_x86_64
pp38-macosx_arm64 | pp38-win_amd64 | pp38-manylinux_x86_64
pp38-manylinux_i686 | pp38-manylinux_aarch64 | | | | +| PyPy3.9 v7.3 | pp39-macosx_x86_64
pp39-macosx_arm64 | pp39-win_amd64 | pp39-manylinux_x86_64
pp39-manylinux_i686 | pp39-manylinux_aarch64 | | | | +| PyPy3.10 v7.3 | pp310-macosx_x86_64
pp310-macosx_arm64 | pp310-win_amd64 | pp310-manylinux_x86_64
pp310-manylinux_i686 | pp310-manylinux_aarch64 | | | | +| PyPy3.11 v7.3 | pp311-macosx_x86_64
pp311-macosx_arm64 | pp311-win_amd64 | pp311-manylinux_x86_64
pp311-manylinux_i686 | pp311-manylinux_aarch64 | | | | | GraalPy 3.11 v24.2 | gp311_242-macosx_x86_64
gp311_242-macosx_arm64 | gp311_242-win_amd64 | gp311_242-manylinux_x86_64 | gp311_242-manylinux_aarch64 | | | The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to cibuildwheel. @@ -190,6 +191,7 @@ Options: - macOS: `x86_64` `arm64` `universal2` - Windows: `AMD64` `x86` `ARM64` - Pyodide: `wasm32` +- Android: `arm64_v8a` `x86_64` - iOS: `arm64_iphoneos` `arm64_iphonesimulator` `x86_64_iphonesimulator` - `auto`: The recommended archs for your machine - see the table below. - `auto64`: The 64-bit arch(s) supported by your machine (includes device and simulator for iOS) @@ -237,7 +239,7 @@ If not listed above, `auto` is the same as `native`. [binfmt]: https://hub.docker.com/r/tonistiigi/binfmt Platform-specific environment variables are also available:
- `CIBW_ARCHS_MACOS` | `CIBW_ARCHS_WINDOWS` | `CIBW_ARCHS_LINUX` | `CIBW_ARCHS_IOS` +`CIBW_ARCHS_MACOS` | `CIBW_ARCHS_WINDOWS` | `CIBW_ARCHS_LINUX` | `CIBW_ARCHS_ANDROID` | `CIBW_ARCHS_IOS` This option can also be set using the [command-line option](#command-line) `--archs`. This option cannot be set in an `overrides` section in `pyproject.toml`. @@ -471,11 +473,11 @@ possible, both through `--installer=uv` passed to build, as well as when making all build and test environments. This will generally speed up cibuildwheel. Make sure you have an external uv on Windows and macOS, either by pre-installing it, or installing cibuildwheel with the uv extra, -`cibuildwheel[uv]`. You cannot use uv currently on Windows for ARM, for -musllinux on s390x, or for iOS, as binaries are not provided by uv. Legacy dependencies like +`cibuildwheel[uv]`. uv currently does not support Windows on ARM, +musllinux on s390x, Android, or iOS. Legacy dependencies like setuptools on Python < 3.12 and pip are not installed if using uv. -Pyodide ignores this setting, as only "build" is supported. +On Android and Pyodide, only "build" is supported. You can specify extra arguments to pass to the build frontend using the optional `args` option. @@ -539,7 +541,7 @@ a table of items, including arrays. single values. Platform-specific environment variables also available:
-`CIBW_CONFIG_SETTINGS_MACOS` | `CIBW_CONFIG_SETTINGS_WINDOWS` | `CIBW_CONFIG_SETTINGS_LINUX` | `CIBW_CONFIG_SETTINGS_IOS` | `CIBW_CONFIG_SETTINGS_PYODIDE` +`CIBW_CONFIG_SETTINGS_MACOS` | `CIBW_CONFIG_SETTINGS_WINDOWS` | `CIBW_CONFIG_SETTINGS_LINUX` | `CIBW_CONFIG_SETTINGS_ANDROID` | `CIBW_CONFIG_SETTINGS_IOS` | `CIBW_CONFIG_SETTINGS_PYODIDE` #### Examples @@ -572,7 +574,7 @@ You can use `$PATH` syntax to insert other variables, or the `$(pwd)` syntax to To specify more than one environment variable, separate the assignments by spaces. Platform-specific environment variables are also available:
-`CIBW_ENVIRONMENT_MACOS` | `CIBW_ENVIRONMENT_WINDOWS` | `CIBW_ENVIRONMENT_LINUX` | `CIBW_ENVIRONMENT_IOS` | `CIBW_ENVIRONMENT_PYODIDE` +`CIBW_ENVIRONMENT_MACOS` | `CIBW_ENVIRONMENT_WINDOWS` | `CIBW_ENVIRONMENT_LINUX` | `CIBW_ENVIRONMENT_ANDROID` | `CIBW_ENVIRONMENT_IOS` | `CIBW_ENVIRONMENT_PYODIDE` #### Examples @@ -704,7 +706,7 @@ On linux, overriding it triggers a new container launch. It cannot be overridden on macOS and Windows. Platform-specific environment variables also available:
-`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` | `CIBW_BEFORE_ALL_IOS` | `CIBW_BEFORE_ALL_PYODIDE` +`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` | `CIBW_BEFORE_ALL_ANDROID` | `CIBW_BEFORE_ALL_IOS` | `CIBW_BEFORE_ALL_PYODIDE` !!! note @@ -770,7 +772,7 @@ The active Python binary can be accessed using `python`, and pip with `pip`; cib The command is run in a shell, so you can write things like `cmd1 && cmd2`. Platform-specific environment variables are also available:
- `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` | `CIBW_BEFORE_BUILD_IOS` | `CIBW_BEFORE_BUILD_PYODIDE` + `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` | `CIBW_BEFORE_BUILD_ANDROID` | `CIBW_BEFORE_BUILD_IOS` | `CIBW_BEFORE_BUILD_PYODIDE` #### Examples @@ -888,9 +890,9 @@ Default: - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` - on macOS: `'delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}'` -- on Windows: `''` -- on iOS: `''` -- on Pyodide: `''` +- on Android: There is no default command, but cibuildwheel will add `libc++` to the + wheel if anything links against it. Setting a command will replace this behavior. +- on other platforms: `''` A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them. The command is run on each built wheel (except for pure Python ones) before testing it. @@ -904,7 +906,7 @@ The following placeholders must be used inside the command and will be replaced The command is run in a shell, so you can run multiple commands like `cmd1 && cmd2`. Platform-specific environment variables are also available:
-`CIBW_REPAIR_WHEEL_COMMAND_MACOS` | `CIBW_REPAIR_WHEEL_COMMAND_WINDOWS` | `CIBW_REPAIR_WHEEL_COMMAND_LINUX` | `CIBW_REPAIR_WHEEL_COMMAND_IOS` | `CIBW_REPAIR_WHEEL_COMMAND_PYODIDE` +`CIBW_REPAIR_WHEEL_COMMAND_MACOS` | `CIBW_REPAIR_WHEEL_COMMAND_WINDOWS` | `CIBW_REPAIR_WHEEL_COMMAND_LINUX` | `CIBW_REPAIR_WHEEL_COMMAND_ANDROID` | `CIBW_REPAIR_WHEEL_COMMAND_IOS` | `CIBW_REPAIR_WHEEL_COMMAND_PYODIDE` !!! tip cibuildwheel doesn't yet ship a default repair command for Windows. @@ -1205,7 +1207,7 @@ specifiers inline with the `packages: SPECIFIER...` syntax. `./constraints.txt` if that's not found. Platform-specific environment variables are also available:
-`CIBW_DEPENDENCY_VERSIONS_MACOS` | `CIBW_DEPENDENCY_VERSIONS_WINDOWS` | `CIBW_DEPENDENCY_VERSIONS_IOS` | `CIBW_DEPENDENCY_VERSIONS_PYODIDE` +`CIBW_DEPENDENCY_VERSIONS_MACOS` | `CIBW_DEPENDENCY_VERSIONS_WINDOWS` | `CIBW_DEPENDENCY_VERSIONS_ANDROID` | `CIBW_DEPENDENCY_VERSIONS_IOS` | `CIBW_DEPENDENCY_VERSIONS_PYODIDE` !!! note This option does not affect the tools used on the Linux build - those versions @@ -1316,8 +1318,8 @@ Shell command to run tests after the build. The wheel will be installed automatically and available for import from the tests. If this variable is not set, your wheel will not be installed after building. -To ensure the wheel is imported by your tests (instead of your source copy), -**Tests are executed from a temporary directory**, outside of your source +To ensure your tests import the wheel (instead of your source tree), +**tests are executed from a temporary working directory**, outside of your source tree. To access your test code, you have a couple of options: - You can use the [`test-sources`](#test-sources) setting to copy specific @@ -1334,14 +1336,20 @@ tree. To access your test code, you have a couple of options: - `{project}` is an absolute path to the project root - the working directory where cibuildwheel was called. -On all platforms other than iOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. + These placeholders are not available on Android and iOS, because those + platforms run tests in a virtual machine that does not have access to + the build machine's filesystem. -On iOS, the value of the `test-command` setting must follow the format `python --m MODULE [ARGS...]` - where MODULE is a Python module name, followed by -arguments that will be assigned to `sys.argv`. Other commands cannot be used. +On all platforms other than Android and iOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. + +On Android and iOS, the command is parsed by `shlex.split`, and is required to +be in one of the following forms: + +* `python -c command ...` (Android only) +* `python -m module-name ...` Platform-specific environment variables are also available:
-`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` | `CIBW_TEST_COMMAND_IOS` | `CIBW_TEST_COMMAND_PYODIDE` +`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` | `CIBW_TEST_COMMAND_ANDROID` | `CIBW_TEST_COMMAND_IOS` | `CIBW_TEST_COMMAND_PYODIDE` #### Examples @@ -1399,7 +1407,7 @@ The active Python binary can be accessed using `python`, and pip with `pip`; cib The command is run in a shell, so you can write things like `cmd1 && cmd2`. Platform-specific environment variables are also available:
- `CIBW_BEFORE_TEST_MACOS` | `CIBW_BEFORE_TEST_WINDOWS` | `CIBW_BEFORE_TEST_LINUX` | `CIBW_BEFORE_TEST_IOS` | `CIBW_BEFORE_TEST_PYODIDE` + `CIBW_BEFORE_TEST_MACOS` | `CIBW_BEFORE_TEST_WINDOWS` | `CIBW_BEFORE_TEST_LINUX` | `CIBW_BEFORE_TEST_ANDROID` | `CIBW_BEFORE_TEST_IOS` | `CIBW_BEFORE_TEST_PYODIDE` #### Examples @@ -1454,20 +1462,15 @@ Platform-specific environment variables are also available:
### `test-sources` {: #test-sources env-var toml} -> Files and folders from the source tree that are copied into an isolated tree before running the tests +> Paths that are copied into the working directory of the tests A space-separated list of files and folders, relative to the root of the project, required for running the tests. If specified, these files and folders -will be copied into a temporary folder, and that temporary folder will be used -as the working directory for running the test suite. - -The use of `test-sources` is *required* for iOS builds. This is because the -simulator does not have access to the project directory, as it is not stored on -the simulator device. On iOS, the files will be copied into the test application, -rather than a temporary folder. +will be copied into the temporary folder which is used as the working directory +for running the test suite. For more details, see [`test-command`](#test-command). Platform-specific environment variables are also available:
-`CIBW_TEST_SOURCES_MACOS` | `CIBW_TEST_SOURCES_WINDOWS` | `CIBW_TEST_SOURCES_LINUX` | `CIBW_TEST_SOURCES_IOS` | `CIBW_TEST_SOURCES_PYODIDE` +`CIBW_TEST_SOURCES_MACOS` | `CIBW_TEST_SOURCES_WINDOWS` | `CIBW_TEST_SOURCES_LINUX` | `CIBW_TEST_SOURCES_ANDROID` | `CIBW_TEST_SOURCES_IOS` | `CIBW_TEST_SOURCES_PYODIDE` #### Examples @@ -1495,7 +1498,7 @@ Platform-specific environment variables are also available:
Space-separated list of dependencies required for running the tests. Platform-specific environment variables are also available:
-`CIBW_TEST_REQUIRES_MACOS` | `CIBW_TEST_REQUIRES_WINDOWS` | `CIBW_TEST_REQUIRES_LINUX` | `CIBW_TEST_REQUIRES_IOS` | `CIBW_TEST_REQUIRES_PYODIDE` +`CIBW_TEST_REQUIRES_MACOS` | `CIBW_TEST_REQUIRES_WINDOWS` | `CIBW_TEST_REQUIRES_LINUX` | `CIBW_TEST_REQUIRES_ANDROID` | `CIBW_TEST_REQUIRES_IOS` | `CIBW_TEST_REQUIRES_PYODIDE` #### Examples @@ -1536,7 +1539,7 @@ tests. This can be used to avoid having to redefine test dependencies in `setup.cfg` or `setup.py`. Platform-specific environment variables are also available:
-`CIBW_TEST_EXTRAS_MACOS` | `CIBW_TEST_EXTRAS_WINDOWS` | `CIBW_TEST_EXTRAS_LINUX` | `CIBW_TEST_EXTRAS_IOS` | `CIBW_TEST_EXTRAS_PYODIDE` +`CIBW_TEST_EXTRAS_MACOS` | `CIBW_TEST_EXTRAS_WINDOWS` | `CIBW_TEST_EXTRAS_LINUX` | `CIBW_TEST_EXTRAS_ANDROID` | `CIBW_TEST_EXTRAS_IOS` | `CIBW_TEST_EXTRAS_PYODIDE` #### Examples @@ -1637,7 +1640,7 @@ A space-separated list of environment variables to set in the test environment. The syntax is the same as for [`environment`](#environment). Platform-specific environment variables are also available:
-`CIBW_TEST_ENVIRONMENT_MACOS` | `CIBW_TEST_ENVIRONMENT_WINDOWS` | `CIBW_TEST_ENVIRONMENT_LINUX` | `CIBW_TEST_ENVIRONMENT_IOS` | `CIBW_TEST_ENVIRONMENT_PYODIDE` +`CIBW_TEST_ENVIRONMENT_MACOS` | `CIBW_TEST_ENVIRONMENT_WINDOWS` | `CIBW_TEST_ENVIRONMENT_LINUX` | `CIBW_TEST_ENVIRONMENT_ANDROID` |`CIBW_TEST_ENVIRONMENT_IOS` | `CIBW_TEST_ENVIRONMENT_PYODIDE` #### Examples @@ -1720,7 +1723,7 @@ The default build frontend is `build`, which does show build backend output by default. Platform-specific environment variables are also available:
-`CIBW_BUILD_VERBOSITY_MACOS` | `CIBW_BUILD_VERBOSITY_WINDOWS` | `CIBW_BUILD_VERBOSITY_LINUX` | `CIBW_BUILD_VERBOSITY_IOS` | `CIBW_BUILD_VERBOSITY_PYODIDE` +`CIBW_BUILD_VERBOSITY_MACOS` | `CIBW_BUILD_VERBOSITY_WINDOWS` | `CIBW_BUILD_VERBOSITY_LINUX` | `CIBW_BUILD_VERBOSITY_ANDROID` | `CIBW_BUILD_VERBOSITY_IOS` | `CIBW_BUILD_VERBOSITY_PYODIDE` #### Examples diff --git a/docs/platforms.md b/docs/platforms.md index 17cc40a88..d67da7a8c 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -178,6 +178,79 @@ If there are pre-releases available for a newer Pyodide version, the `pyodide-pr Currently, it's recommended to run tests using a `python -m` entrypoint, rather than a command line entrypoint, or a shell script. This is because custom entrypoints have some issues in the Pyodide virtual environment. For example, `pytest` may not work as a command line entrypoint, but will work as a `python -m pytest` entrypoint. + +## Android {: android} + +### Prerequisites + +cibuildwheel can build Android wheels on any POSIX platform supported by the Android +development tools, which currently means Linux x86_64, macOS ARM64 or macOS x86_64. Any +of these platforms can be used to build wheels for any Android architecture supported by +Python. However, *testing* wheels has additional requirements: see the section below. + +If you already have an Android SDK, export the `ANDROID_HOME` environment variable to +point at its location. Otherwise, here's how to install it: + +* Download the "Command line tools" from . +* Create a directory `android-sdk/cmdline-tools`, and unzip the command line + tools package into it. +* Rename `android-sdk/cmdline-tools/cmdline-tools` to + `android-sdk/cmdline-tools/latest`. +* `export ANDROID_HOME=/path/to/android-sdk` + +cibuildwheel will automatically use the SDK's `sdkmanager` to install any packages it +needs. + +It also requires the following commands to be on the `PATH`: + +* `curl` +* `java` (or set the `JAVA_HOME` environment variable) +* `patchelf` (if the wheel links against any external libraries) + +### Android version compatibility + +Android builds will honor the `ANDROID_API_LEVEL` environment variable to set the +minimum supported [API level](https://developer.android.com/tools/releases/platforms) +for generated wheels. This will default to the minimum API level of the selected Python +version. + +### Build frontend support + +Android builds only support the `build` frontend. In principle, support for the +`build[uv]` frontend should be possible, but `uv` [doesn't currently have support for +cross-platform builds](https://github.com/astral-sh/uv/issues/7957), and [doesn't have +support for iOS or Android wheel tags](https://github.com/astral-sh/uv/issues/8029). + +### Tests + +Tests are executed on a Gradle-managed emulator matching the architecture of the build +machine – for example, if you're building on an ARM64 machine, then you can test an +ARM64 wheel. Wheels of other architectures can still be built, but testing will +automatically be skipped. + +Running an emulator requires the build machine to either be bare-metal or support +nested virtualization. CI platforms known to meet this requirement are: + +* GitHub Actions Linux x86_64 + +On Linux, the emulator needs access to the KVM virtualization interface. This may +require adding your user to a group, or [changing your udev +rules](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). +If the emulator fails to start, try running `$ANDROID_HOME/emulator/emulator +-accel-check`. + +The Android test environment can't support running shell scripts, so the +[`test-command`](options.md#test-command) must be a Python command – see its +documentation for details. + +If your package has dependencies which haven't been released on PyPI yet, you may want +to use the [`environment`](options.md#environment) option to set `PIP_EXTRA_INDEX_URL` +to one of the following URLs: + +* https://chaquo.com/pypi-13.1 +* https://pypi.anaconda.org/scientific-python-nightly-wheels/simple + + ## iOS {: #ios} ### System requirements @@ -241,6 +314,6 @@ If your project requires additional tools to build (such as `cmake`, `ninja`, or If tests have been configured, the test suite will be executed on the simulator matching the architecture of the build machine - that is, if you're building on an ARM64 macOS machine, the ARM64 wheel will be tested on an ARM64 simulator. It is not possible to use cibuildwheel to test wheels on other simulators, or on physical devices. -The iOS test environment can't support running shell scripts, so the [`test-command`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. In addition, the project must use [`test-sources`](options.md#test-sources) to specify the minimum subset of files that should be copied to the test environment. This is because the test must be run "on device", and the simulator device will not have access to the local project directory. +The iOS test environment can't support running shell scripts, so the [`test-command`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. The test process uses the same testbed used by CPython itself to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index 8713fee90..134020bc5 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -31,6 +31,12 @@ jobs: - os: macos-arm # macos-14+ (including latest) are ARM64 runners runs-on: macos-latest + - os: android-intel + runs-on: ubuntu-latest + platform: android + - os: android-arm + runs-on: macos-latest + platform: android - os: ios runs-on: macos-latest platform: ios @@ -41,6 +47,19 @@ jobs: steps: - uses: actions/checkout@v4 + # GitHub Actions can't currently run the Android emulator on macOS. + - name: Skip Android tests on macOS + if: matrix.os == 'android-arm' + run: echo "CIBW_TEST_COMMAND=" >> "$GITHUB_ENV" + + # https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/ + - name: Enable KVM for Android emulator + if: matrix.os == 'android-intel' + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Build wheels uses: pypa/cibuildwheel@v3.0.1 env: diff --git a/pyproject.toml b/pyproject.toml index 55b254b0f..abe0a92eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,12 +39,15 @@ classifiers = [ dependencies = [ "bashlex!=0.13", "bracex", + "build>=1.0.0", "certifi", "dependency-groups>=1.2", "filelock", "humanize", "packaging>=20.9", - "platformdirs" + "platformdirs", + "pyelftools>=0.29", + "wheel>=0.33.6", ] [project.optional-dependencies] diff --git a/test/test_android.py b/test/test_android.py new file mode 100644 index 000000000..d176b65bd --- /dev/null +++ b/test/test_android.py @@ -0,0 +1,383 @@ +import os +import platform +import re +from dataclasses import dataclass +from shutil import rmtree +from subprocess import CalledProcessError +from textwrap import dedent +from zipfile import ZipFile + +import pytest + +from .test_projects import new_c_project +from .utils import cibuildwheel_run, expected_wheels + +CIBW_PLATFORM = os.environ.get("CIBW_PLATFORM", "android") +if CIBW_PLATFORM != "android": + pytest.skip(f"{CIBW_PLATFORM=}", allow_module_level=True) + +if (platform.system(), platform.machine()) not in [ + ("Linux", "x86_64"), + ("Darwin", "arm64"), + ("Darwin", "x86_64"), +]: + pytest.skip( + f"cibuildwheel does not support building Android wheels on " + f"{platform.system()} {platform.machine()}", + allow_module_level=True, + ) + +# Detect CI services which have the Android SDK pre-installed. +ci_supports_build = any( + key in os.environ + for key in [ + "GITHUB_ACTIONS", + "TF_BUILD", # Azure Pipelines + ] +) + +if "ANDROID_HOME" not in os.environ: + msg = "ANDROID_HOME environment variable is not set" + if ci_supports_build: + pytest.fail(msg) + else: + pytest.skip(msg, allow_module_level=True) + +# Many CI services don't support running the Android emulator: see platforms.md. +ci_supports_emulator = "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux" + + +def needs_emulator(test): + # All copies of the testbed app run on the same emulator with the same + # application ID, so these tests must be run serially. + test = pytest.mark.serial(test) + + if ci_supports_build and not ci_supports_emulator: + test = pytest.mark.skip("This CI platform doesn't support the emulator")(test) + return test + + +@dataclass +class Architecture: + linux_machine: str + macos_machine: str + android_abi: str + + +archs = [ + Architecture("aarch64", "arm64", "arm64_v8a"), + Architecture("x86_64", "x86_64", "x86_64"), +] +native_arch = next( + arch for arch in archs if platform.machine() in [arch.linux_machine, arch.macos_machine] +) + + +cp313_env = { + "CIBW_PLATFORM": "android", + "CIBW_BUILD": "cp313-*", +} + + +def test_android_home(tmp_path, capfd): + new_c_project().generate(tmp_path) + env = os.environ.copy() + del env["ANDROID_HOME"] + + with pytest.raises(CalledProcessError): + cibuildwheel_run(tmp_path, env={**env, **cp313_env}) + assert "ANDROID_HOME environment variable is not set" in capfd.readouterr().err + + +def test_frontend_good(tmp_path): + new_c_project().generate(tmp_path) + wheels = cibuildwheel_run( + tmp_path, + add_env={**cp313_env, "CIBW_BUILD_FRONTEND": "build"}, + ) + assert wheels == [f"spam-0.1.0-cp313-cp313-android_21_{native_arch.android_abi}.whl"] + + +@pytest.mark.parametrize("frontend", ["build[uv]", "pip"]) +def test_frontend_bad(frontend, tmp_path, capfd): + new_c_project().generate(tmp_path) + with pytest.raises(CalledProcessError): + cibuildwheel_run( + tmp_path, + add_env={**cp313_env, "CIBW_BUILD_FRONTEND": frontend}, + ) + assert "Android requires the build frontend to be 'build'" in capfd.readouterr().err + + +def test_expected_wheels(tmp_path): + new_c_project().generate(tmp_path) + wheels = cibuildwheel_run(tmp_path, add_env={"CIBW_PLATFORM": "android"}) + assert wheels == expected_wheels( + "spam", "0.1.0", platform="android", machine_arch=native_arch.android_abi + ) + + +@needs_emulator +def test_archs(tmp_path, capfd): + new_c_project().generate(tmp_path) + + # Build all architectures while checking the handling of the `before` commands. + command_pattern = 'echo "Hello from {0}, package={{package}}, python=$(which python)"' + output_pattern = ( + f"Hello from {{0}}, package={tmp_path}, python=/.+/cp313-android_{{1}}/venv/bin/python" + ) + + wheels = cibuildwheel_run( + tmp_path, + add_env={ + **cp313_env, + "CIBW_ARCHS": "all", + "CIBW_BEFORE_ALL": "echo 'Hello from before_all'", + "CIBW_BEFORE_BUILD": command_pattern.format("before_build"), + "CIBW_BEFORE_TEST": command_pattern.format("before_test"), + "CIBW_TEST_COMMAND": ( + "python -c 'import platform; print(f\"Hello from {platform.machine()}\")'" + ), + }, + ) + assert wheels == [f"spam-0.1.0-cp313-cp313-android_21_{arch.android_abi}.whl" for arch in archs] + + stdout, stderr = capfd.readouterr() + lines = (line for line in stdout.splitlines() if line.startswith("Hello from")) + assert next(lines) == "Hello from before_all" + + # All architectures should be built, but only the native architecture should run tests. + for arch in archs: + abi = arch.android_abi + assert re.fullmatch(output_pattern.format("before_build", abi), next(lines)) + if arch == native_arch: + assert re.fullmatch(output_pattern.format("before_test", abi), next(lines)) + assert next(lines) == f"Hello from {arch.linux_machine}" + else: + assert ( + f"Skipping tests for {arch.android_abi}, as the build machine " + f"only supports {native_arch.android_abi}" + ) in stderr + + try: + line = next(lines) + except StopIteration: + pass + else: + pytest.fail(f"Unexpected line: {line!r}") + + +def test_build_requires(tmp_path, capfd): + # Build-time requirements should be installed for the build platform, not for Android. Prove + # this by installing some non-pure-Python requirements and using them in setup.py. + # + # setup_requires is installed via ProjectBuilder.get_requires_for_build. + project = new_c_project( + setup_py_setup_args_add="setup_requires=['cmake==3.31.4']", + setup_py_add=dedent( + """\ + if "egg_info" not in sys.argv: + import subprocess + subprocess.run(["cmake", "--version"], check=True) + + from bitarray import bitarray + print(f"{bitarray('10110').count()=}") + """ + ), + ) + + # [build_system] requires is installed via ProjectBuilder.build_system_requires. + project.files["pyproject.toml"] = dedent( + """\ + [build-system] + requires = ["setuptools", "wheel", "bitarray==3.3.2"] + """ + ) + + project.generate(tmp_path) + cibuildwheel_run(tmp_path, add_env={**cp313_env}) + + # Test for a specific version to minimize the chance that we ran a system cmake. + stdout = capfd.readouterr().out + assert "cmake version 3.31.4" in stdout + assert "bitarray('10110').count()=3" in stdout + + +@pytest.fixture +def spam_env(tmp_path): + project = new_c_project() + project.files["test_spam.py"] = dedent( + """\ + import spam + + def test_spam(): + assert spam.filter("ham") + assert not spam.filter("spam") + print("Spam test passed") + """ + ) + project.generate(tmp_path) + + return { + **cp313_env, + "CIBW_TEST_SOURCES": "test_spam.py", + "CIBW_TEST_REQUIRES": "pytest==8.3.5", + } + + +@needs_emulator +@pytest.mark.parametrize( + ("command", "expected_output"), + [ + ("python -c 'import test_spam; test_spam.test_spam()'", "Spam test passed"), + ("python -m pytest test_spam.py", "=== 1 passed in "), + ("pytest test_spam.py", "=== 1 passed in "), + ], +) +def test_test_command_good(command, expected_output, tmp_path, spam_env, capfd): + cibuildwheel_run(tmp_path, add_env={**spam_env, "CIBW_TEST_COMMAND": command}) + stdout, stderr = capfd.readouterr() + assert expected_output in stdout + + if not command.startswith("python"): + assert ( + f"Test command {command!r} is not supported on Android. cibuildwheel " + "will try to execute it as if it started with 'python -m'." + ) in stderr + + +@needs_emulator +@pytest.mark.parametrize( + ("command", "expected_output"), + [ + # Build-time failure: unrecognized command + ( + "./test_spam.py", + "Test command './test_spam.py' is not supported on Android. " + "Supported commands are 'python -m' and 'python -c'.", + ), + # Build-time failure: unrecognized placeholder + ( + "pytest {project}", + "Test command 'pytest {project}' with a '{project}' or '{package}' " + "placeholder is not supported on Android", + ), + ( + "pytest {package}", + "Test command 'pytest {package}' with a '{project}' or '{package}' " + "placeholder is not supported on Android", + ), + # Runtime failure + ("pytest test_ham.py", "not found: test_ham.py"), + ], +) +def test_test_command_bad(command, expected_output, tmp_path, spam_env, capfd): + with pytest.raises(CalledProcessError): + cibuildwheel_run(tmp_path, add_env={**spam_env, "CIBW_TEST_COMMAND": command}) + assert expected_output in capfd.readouterr().err + + +@needs_emulator +def test_package_subdir(tmp_path, spam_env, capfd): + spam_paths = list(tmp_path.iterdir()) + package_dir = tmp_path / "package" + package_dir.mkdir() + for path in spam_paths: + path.rename(package_dir / path.name) + + test_filename = "package/" + spam_env["CIBW_TEST_SOURCES"] + cibuildwheel_run( + tmp_path, + package_dir, + add_env={ + **spam_env, + "CIBW_TEST_SOURCES": test_filename, + "CIBW_TEST_COMMAND": f"python -m pytest {test_filename}", + }, + ) + assert "=== 1 passed in " in capfd.readouterr().out + + +@needs_emulator +def test_no_test_sources(tmp_path, capfd): + new_c_project().generate(tmp_path) + with pytest.raises(CalledProcessError): + cibuildwheel_run( + tmp_path, + add_env={**cp313_env, "CIBW_TEST_COMMAND": "python -m unittest discover"}, + ) + assert ( + "On this platform, you must copy your test files to the testbed app by " + "setting the `test-sources` option" + ) in capfd.readouterr().err + + +@needs_emulator +def test_api_level(tmp_path, capfd): + project = new_c_project() + project.files["pyproject.toml"] = dedent( + """\ + [build-system] + requires = ["setuptools"] + + [tool.cibuildwheel] + android.environment.ANDROID_API_LEVEL = "33" + android.environment.PIP_EXTRA_INDEX_URL = "https://chaquo.com/pypi-13.1" + """ + ) + project.generate(tmp_path) + + wheels = cibuildwheel_run( + tmp_path, + add_env={ + **cp313_env, + # Verify that Android dependencies can be installed from the Chaquopy repository, and + # that wheels tagged with an older version of Android (in this case 24) are still + # accepted. + "CIBW_TEST_REQUIRES": "bitarray==3.0.0", + "CIBW_TEST_COMMAND": ( + "python -c 'from bitarray import bitarray; print(~bitarray(\"01100\"))'" + ), + }, + ) + assert wheels == [f"spam-0.1.0-cp313-cp313-android_33_{native_arch.android_abi}.whl"] + assert "bitarray('10011')" in capfd.readouterr().out + + +@needs_emulator +def test_libcxx(tmp_path, capfd): + project_dir = tmp_path / "project" + output_dir = tmp_path / "output" + + # A C++ package should include libc++, and the extension module should be able to + # find it using DT_RUNPATH. + new_c_project(setup_py_extension_args_add="language='c++'").generate(project_dir) + script = 'import spam; print(", ".join(f"{s}: {spam.filter(s)}" for s in ["ham", "spam"]))' + cp313_test_env = {**cp313_env, "CIBW_TEST_COMMAND": f"python -c '{script}'"} + + # Including external libraries requires API level 24. + with pytest.raises(CalledProcessError): + cibuildwheel_run(project_dir, add_env=cp313_test_env, output_dir=output_dir) + assert "libc++_shared.so requires ANDROID_API_LEVEL to be at least 24" in capfd.readouterr().err + + wheels = cibuildwheel_run( + project_dir, + add_env={**cp313_test_env, "ANDROID_API_LEVEL": "24"}, + output_dir=output_dir, + ) + assert len(wheels) == 1 + names = ZipFile(output_dir / wheels[0]).namelist() + libcxx_names = [ + name for name in names if re.fullmatch(r"spam\.libs/libc\+\+_shared-[0-9a-f]{8}\.so", name) + ] + assert len(libcxx_names) == 1 + assert "ham: 1, spam: 0" in capfd.readouterr().out + + # A C package should not include libc++. + rmtree(project_dir) + rmtree(output_dir) + new_c_project().generate(project_dir) + wheels = cibuildwheel_run(project_dir, add_env=cp313_env, output_dir=output_dir) + assert len(wheels) == 1 + for name in ZipFile(output_dir / wheels[0]).namelist(): + assert ".libs" not in name diff --git a/test/test_projects/c.py b/test/test_projects/c.py index 55d759875..9643e4cda 100644 --- a/test/test_projects/c.py +++ b/test/test_projects/c.py @@ -17,7 +17,7 @@ return NULL; // Spam should not be allowed through the filter. - sts = strcmp(content, "spam"); + sts = strcmp(content, "spam") != 0; {{ spam_c_function_add | indent(4) }} diff --git a/test/utils.py b/test/utils.py index ccd467aa9..b5d4dcf99 100644 --- a/test/utils.py +++ b/test/utils.py @@ -151,7 +151,7 @@ def cibuildwheel_run( cwd=project_path, check=True, ) - wheels = [p.name for p in (output_dir or Path(tmp_output_dir)).iterdir()] + wheels = sorted(p.name for p in (output_dir or Path(tmp_output_dir)).iterdir()) return wheels @@ -262,7 +262,7 @@ def _expected_wheels( if platform == "pyodide" and python_abi_tags is None: python_abi_tags = ["cp312-cp312", "cp313-cp313"] - elif platform == "ios" and python_abi_tags is None: + elif platform in {"android", "ios"} and python_abi_tags is None: python_abi_tags = ["cp313-cp313"] elif python_abi_tags is None: python_abi_tags = [ @@ -368,6 +368,13 @@ def _expected_wheels( if include_universal2: platform_tags.append(f"macosx_{min_macosx.replace('.', '_')}_universal2") + elif platform == "android": + api_level = { + "cp313-cp313": 21, + "cp314-cp314": 24, + }[python_abi_tag] + platform_tags = [f"android_{api_level}_{machine_arch}"] + elif platform == "ios": if machine_arch == "x86_64": platform_tags = [ diff --git a/unit_test/architecture_test.py b/unit_test/architecture_test.py index 98efe91d8..3033e2be9 100644 --- a/unit_test/architecture_test.py +++ b/unit_test/architecture_test.py @@ -5,7 +5,7 @@ import pytest import cibuildwheel.architecture -from cibuildwheel.architecture import Architecture +from cibuildwheel.architecture import Architecture, arch_synonym @pytest.fixture( @@ -115,3 +115,20 @@ def test_arch_auto_on_ios(monkeypatch): monkeypatch.setattr(platform_module, "machine", lambda: "arm64") arch_set = Architecture.parse_config("auto", platform="ios") assert arch_set == {Architecture.arm64_iphonesimulator, Architecture.arm64_iphoneos} + + +@pytest.mark.parametrize( + ("arch", "from_platform", "to_platform", "expected"), + [ + ("x86_64", "linux", "macos", "x86_64"), + ("x86_64", "macos", "linux", "x86_64"), + ("x86_64", "linux", "windows", "AMD64"), + ("AMD64", "windows", "linux", "x86_64"), + ("x86_64", "linux", "nonexistent", "x86_64"), + ("x86_64", "nonexistent", "linux", "x86_64"), + ("nonexistent", "linux", "windows", "nonexistent"), + ("x86", "windows", "macos", None), + ], +) +def test_arch_synonym(arch, from_platform, to_platform, expected): + assert arch_synonym(arch, from_platform, to_platform) == expected diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index cf421ce49..ab6eb4178 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -7,7 +7,7 @@ from cibuildwheel.__main__ import main from cibuildwheel.environment import ParsedEnvironment -from cibuildwheel.frontend import _split_config_settings +from cibuildwheel.frontend import _split_config_settings, parse_config_settings from cibuildwheel.options import BuildOptions, _get_pinned_container_images from cibuildwheel.selector import BuildSelector, EnableGroup from cibuildwheel.util import resources @@ -289,7 +289,9 @@ def test_build_verbosity( @pytest.mark.parametrize("platform_specific", [False, True]) def test_config_settings(platform_specific, platform, intercepted_build_args, monkeypatch): - config_settings = 'setting=value setting=value2 other="something else"' + config_settings = ( + 'setting=value setting=value2 triplet=1 triplet=2 triplet=3 other="something else"' + ) if platform_specific: monkeypatch.setenv("CIBW_CONFIG_SETTINGS_" + platform.upper(), config_settings) monkeypatch.setenv("CIBW_CONFIG_SETTINGS", "a=b") @@ -304,8 +306,16 @@ def test_config_settings(platform_specific, platform, intercepted_build_args, mo assert _split_config_settings(config_settings) == [ "-Csetting=value", "-Csetting=value2", + "-Ctriplet=1", + "-Ctriplet=2", + "-Ctriplet=3", "-Cother=something else", ] + assert parse_config_settings(config_settings) == { + "setting": ["value", "value2"], + "triplet": ["1", "2", "3"], + "other": "something else", + } @pytest.mark.parametrize( @@ -494,7 +504,8 @@ def test_defaults(platform, intercepted_build_args): if isinstance(repair_wheel_default, list): repair_wheel_default = " && ".join(repair_wheel_default) assert build_options.repair_command == repair_wheel_default - assert build_options.build_frontend is None + assert build_options.build_frontend.name == "build" + assert build_options.build_frontend.args == () if platform == "linux": assert build_options.manylinux_images diff --git a/unit_test/options_test.py b/unit_test/options_test.py index d1207aa15..52f2f1adf 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -363,11 +363,11 @@ def test_build_frontend_option( parsed_build_frontend = options.build_options(identifier=None).build_frontend if toml_assignment: - assert parsed_build_frontend is not None assert parsed_build_frontend.name == result_name assert parsed_build_frontend.args == result_args else: - assert parsed_build_frontend is None + assert parsed_build_frontend.name == "build" + assert parsed_build_frontend.args == () def test_override_inherit_environment(tmp_path: Path) -> None: