From 2ae389d670ec22b9a84e4a9d785a94d2ea03a4bc Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 11 Sep 2024 02:17:57 +0530 Subject: [PATCH 001/109] =?UTF-8?q?Fix=20a=20typo:=20pyoodide=20=E2=9E=A1?= =?UTF-8?q?=EF=B8=8F=20pyodide?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/test_emscripten.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_emscripten.py b/test/test_emscripten.py index 97ceb7f9f..aa7750bc5 100644 --- a/test/test_emscripten.py +++ b/test/test_emscripten.py @@ -20,7 +20,7 @@ def check_node(): # cibuildwheel adds a pinned node version to the PATH - # check it's in the PATH then, check it's the one that runs pyoodide + # check it's in the PATH then, check it's the one that runs pyodide cibw_cache_path = Path(sys.argv[1]).resolve(strict=True) # find the node executable in PATH node = shutil.which("node") From 0e3b3f1b8ae6d19cb3a2a4383da31b78d3e51228 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 11 Sep 2024 03:12:32 +0530 Subject: [PATCH 002/109] Add `pyodide_build_version` attribute --- cibuildwheel/pyodide.py | 13 ++++++++++--- cibuildwheel/resources/build-platforms.toml | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 3b721a10a..eba3546e1 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -41,6 +41,7 @@ class PythonConfiguration: version: str identifier: str pyodide_version: str + pyodide_build_version: str emscripten_version: str node_version: str @@ -65,10 +66,14 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path -def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str: +def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: + # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are + # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall + # specify the pyodide-build version, which will set up the xbuildenv for the requested + # Pyodide version. pyodide_root = ( CIBW_CACHE_PATH - / f".pyodide-xbuildenv-{pyodide_version}/{pyodide_version}/xbuildenv/pyodide-root" + / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" ) with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): if pyodide_root.exists(): @@ -171,7 +176,9 @@ def setup_python( env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) log.step("Installing Pyodide xbuildenv...") - env["PYODIDE_ROOT"] = install_xbuildenv(env, python_configuration.pyodide_version) + env["PYODIDE_ROOT"] = install_xbuildenv( + env, python_configuration.pyodide_build_version, python_configuration.pyodide_version + ) return env diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index f96e6897a..1c967c9f5 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -172,5 +172,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.28.0", emscripten_version = "3.1.58", node_version = "v20" }, ] From 394459fe359b26afab787f026dda7772b998c9ee Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 11 Sep 2024 03:28:26 +0530 Subject: [PATCH 003/109] Add version to xbuildenv log step --- cibuildwheel/pyodide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index eba3546e1..133ece0a0 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -175,7 +175,7 @@ def setup_python( env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) - log.step("Installing Pyodide xbuildenv...") + log.step(f"Installing Pyodide xbuildenv version: {python_configuration.pyodide_version} ...") env["PYODIDE_ROOT"] = install_xbuildenv( env, python_configuration.pyodide_build_version, python_configuration.pyodide_version ) From dff7bf265ac0fb894c6cf35f4851daff3adc1c65 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 11 Sep 2024 03:39:41 +0530 Subject: [PATCH 004/109] Add version to Emscripten log step --- cibuildwheel/pyodide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 133ece0a0..4b8bef432 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -170,7 +170,7 @@ def setup_python( env=env, ) - log.step("Installing emscripten...") + log.step(f"Installing Emscripten version: {python_configuration.emscripten_version} ...") emcc_path = install_emscripten(tmp, python_configuration.emscripten_version) env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) From 16057bca6e211cd282f6981871252110d342f373 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 11 Sep 2024 03:39:04 +0530 Subject: [PATCH 005/109] Use `pyodide-build`'s version for updating constraints --- noxfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 0c2057b6d..4f59a98d8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -102,10 +102,10 @@ def update_constraints(session: nox.Session) -> None: pyodides = build_platforms["pyodide"]["python_configurations"] for pyodide in pyodides: python_version = ".".join(pyodide["version"].split(".")[:2]) - pyodide_version = pyodide["pyodide_version"] + pyodide_build_version = pyodide["pyodide_build_version"] output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt" tmp_file = Path(session.create_tmp()) / "constraints-pyodide.in" - tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_version}") + tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_build_version}") session.run( "uv", "pip", From f167c50cf570790a11dd4e608978321bb7f742cb Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 11 Sep 2024 03:38:23 +0530 Subject: [PATCH 006/109] Bump Pyodide constraints by updating `pyodide-build` --- .../resources/constraints-pyodide312.txt | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 890a0736e..57bcb2c89 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -6,7 +6,7 @@ anyio==4.4.0 # via httpx auditwheel-emscripten==0.0.16 # via pyodide-build -build==1.2.1 +build==1.2.2 # via # -r .nox/update_constraints/tmp/constraints-pyodide.in # pyodide-build @@ -19,13 +19,11 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via typer -cloudpickle==3.0.0 - # via loky -cmake==3.30.2 +cmake==3.30.3 # via pyodide-build distlib==0.3.8 # via virtualenv -filelock==3.15.4 +filelock==3.16.0 # via virtualenv h11==0.14.0 # via httpcore @@ -40,8 +38,6 @@ idna==3.8 # requests leb128==1.0.8 # via auditwheel-emscripten -loky==3.4.1 - # via pyodide-build markdown-it-py==3.0.0 # via rich mdurl==0.1.2 @@ -54,33 +50,31 @@ packaging==24.1 # unearth pip==24.2 # via -r .nox/update_constraints/tmp/constraints-pyodide.in -platformdirs==4.2.2 +platformdirs==4.3.2 # via virtualenv -pydantic==2.8.2 +pydantic==2.9.1 # via # pyodide-build # pyodide-lock -pydantic-core==2.20.1 +pydantic-core==2.23.3 # via pydantic pygments==2.18.0 # via rich -pyodide-build==0.26.1 +pyodide-build==0.28.0 # via -r .nox/update_constraints/tmp/constraints-pyodide.in pyodide-cli==0.2.4 # via # auditwheel-emscripten # pyodide-build -pyodide-lock==0.1.0a6 +pyodide-lock==0.1.0a7 # via pyodide-build pyproject-hooks==1.1.0 # via build -pyyaml==6.0.2 - # via pyodide-build requests==2.32.3 # via pyodide-build resolvelib==1.0.1 # via pyodide-build -rich==13.8.0 +rich==13.8.1 # via # pyodide-build # pyodide-cli @@ -100,8 +94,6 @@ typer==0.12.5 # auditwheel-emscripten # pyodide-build # pyodide-cli -types-requests==2.32.0.20240712 - # via pyodide-build typing-extensions==4.12.2 # via # pydantic @@ -110,10 +102,8 @@ typing-extensions==4.12.2 unearth==0.17.2 # via pyodide-build urllib3==2.2.2 - # via - # requests - # types-requests -virtualenv==20.26.3 + # via requests +virtualenv==20.26.4 # via # build # pyodide-build From 31a6be9557e91bac8c1ad7ac8090f139017c47dd Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 12 Sep 2024 18:49:36 +0530 Subject: [PATCH 007/109] Add a schema for `pyodide-version` --- cibuildwheel/resources/cibuildwheel.schema.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 976751a55..714c73067 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -814,6 +814,11 @@ }, "test-requires": { "$ref": "#/properties/test-requires" + }, + "pyodide-version": { + "description": "Specify the Pyodide xbuildenv version to use for building", + "type": "string", + "title": "CIBW_PYODIDE_VERSION" } } } From 90030679658b4e761f8a2f0c66332db7f9c72d85 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 13 Sep 2024 05:09:43 +0530 Subject: [PATCH 008/109] Update Pyodide constraints --- .../resources/constraints-pyodide312.txt | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index a86e06e9a..25a4835c0 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -19,8 +19,6 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via typer -cloudpickle==3.0.0 - # via loky cmake==3.30.3 # via pyodide-build distlib==0.3.8 @@ -54,11 +52,11 @@ pip==24.2 # via -r .nox/update_constraints/tmp/constraints-pyodide.in platformdirs==4.3.2 # via virtualenv -pydantic==2.9.0 +pydantic==2.9.1 # via # pyodide-build # pyodide-lock -pydantic-core==2.23.2 +pydantic-core==2.23.3 # via pydantic pygments==2.18.0 # via rich @@ -96,21 +94,15 @@ typer==0.12.5 # auditwheel-emscripten # pyodide-build # pyodide-cli -types-requests==2.32.0.20240907 - # via pyodide-build typing-extensions==4.12.2 # via # pydantic # pydantic-core # typer -tzdata==2024.1 - # via pydantic unearth==0.17.2 # via pyodide-build -urllib3==2.2.2 - # via - # requests - # types-requests +urllib3==2.2.3 + # via requests virtualenv==20.26.4 # via # build From d8a8d5e8d73267042e8e0743779029516947fca6 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 21:40:47 +0530 Subject: [PATCH 009/109] Bump `pyodide-build` to new 0.29.0 --- .../resources/constraints-pyodide312.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 25a4835c0..2c80f2f9c 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -2,7 +2,7 @@ # nox -s update_constraints annotated-types==0.7.0 # via pydantic -anyio==4.4.0 +anyio==4.6.0 # via httpx auditwheel-emscripten==0.0.16 # via pyodide-build @@ -23,7 +23,7 @@ cmake==3.30.3 # via pyodide-build distlib==0.3.8 # via virtualenv -filelock==3.16.0 +filelock==3.16.1 # via virtualenv h11==0.14.0 # via httpcore @@ -31,7 +31,7 @@ httpcore==1.0.5 # via httpx httpx==0.27.2 # via unearth -idna==3.8 +idna==3.10 # via # anyio # httpx @@ -50,17 +50,17 @@ packaging==24.1 # unearth pip==24.2 # via -r .nox/update_constraints/tmp/constraints-pyodide.in -platformdirs==4.3.2 +platformdirs==4.3.6 # via virtualenv -pydantic==2.9.1 +pydantic==2.9.2 # via # pyodide-build # pyodide-lock -pydantic-core==2.23.3 +pydantic-core==2.23.4 # via pydantic pygments==2.18.0 # via rich -pyodide-build==0.28.0 +pyodide-build==0.29.0 # via -r .nox/update_constraints/tmp/constraints-pyodide.in pyodide-cli==0.2.4 # via @@ -103,7 +103,7 @@ unearth==0.17.2 # via pyodide-build urllib3==2.2.3 # via requests -virtualenv==20.26.4 +virtualenv==20.26.5 # via # build # pyodide-build From cf5dd3ec9ac65c6e7121ea6d8d4d58158df634bd Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 21:41:00 +0530 Subject: [PATCH 010/109] Test out another Pyodide identifier --- cibuildwheel/resources/build-platforms.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 03fccc9d9..30d5f5a3c 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -172,5 +172,6 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.28.0", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.27.0a2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, ] From 4fe86e0bdc90aaa9bb4afc6d66f53f1b0daec814 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 21:47:34 +0530 Subject: [PATCH 011/109] Update outdated Pyodide constraints --- cibuildwheel/resources/constraints-pyodide312.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index ebb1c6305..2c80f2f9c 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -94,8 +94,6 @@ typer==0.12.5 # auditwheel-emscripten # pyodide-build # pyodide-cli -types-requests==2.32.0.20240914 - # via pyodide-build typing-extensions==4.12.2 # via # pydantic @@ -104,9 +102,7 @@ typing-extensions==4.12.2 unearth==0.17.2 # via pyodide-build urllib3==2.2.3 - # via - # requests - # types-requests + # via requests virtualenv==20.26.5 # via # build From b6830ee4dad18274d60e071655142b70444c971d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 22:25:47 +0530 Subject: [PATCH 012/109] Add Pyodide version to temp directory name --- cibuildwheel/pyodide.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 4b8bef432..7253202a5 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -226,7 +226,8 @@ def build(options: Options, tmp_path: Path) -> None: log.build_start(config.identifier) - identifier_tmp_dir = tmp_path / config.identifier + # Include both the identifier and the Pyodide version in the temp directory name + identifier_tmp_dir = tmp_path / f"{config.identifier}_{config.pyodide_version}" built_wheel_dir = identifier_tmp_dir / "built_wheel" repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" identifier_tmp_dir.mkdir() From aaf32e594d44c1259bc227cbd5e14ac014e8d8c5 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 22:34:58 +0530 Subject: [PATCH 013/109] Remove Pyodide 0.26.1 from build configurations --- cibuildwheel/resources/build-platforms.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 30d5f5a3c..15a711f54 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -172,6 +172,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.27.0a2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, ] From bb6e0d67f85f464ab2ffcf0c3c743b3855b67b4d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 22:57:25 +0530 Subject: [PATCH 014/109] Retrieve + validate + install specific xbuildenvs --- cibuildwheel/pyodide.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 7253202a5..50e0be661 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json import os import shutil import sys @@ -66,6 +67,36 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path +def retrieve_compatible_xbuildenvs() -> list[str]: + """Search for compatible xbuildenvs for the current pyodide-build version.""" + xbuildenvs = call( + "pyodide", + "xbuildenv", + "search", + "--json", + "--all", + capture_stdout=True, + ).strip() + + xbuildenvs_dict = json.loads(xbuildenvs) + compatible_xbuildenvs = [ + env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] + ] + + return compatible_xbuildenvs + + +def validate_xbuildenv_version(pyodide_build_version: str, xbuildenv_version: str) -> None: + """Validate that the requested xbuildenv version is compatible with the pyodide-build version.""" + xbuildenvs = retrieve_compatible_xbuildenvs() + if xbuildenv_version not in xbuildenvs: + msg = ( + f"The xbuildenv version {xbuildenv_version} is not compatible with the pyodide-build version {pyodide_build_version}." + f" The compatible versions are: {xbuildenvs}. Please use the 'pyodide xbuildenv search' command to find the compatible versions." + ) + raise errors.FatalError(msg) + + def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall @@ -75,6 +106,16 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v CIBW_CACHE_PATH / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" ) + + # Validate that the requested xbuildenv version is compatible with the pyodide-build version. + # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, + # which will always be compatible. Hence, this function really only checks for the case where the + # version is supplied manually through CIBW_PYODIDE_VERSION environment variable. + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION") + if cibw_pyodide_version: + validate_xbuildenv_version(pyodide_build_version, cibw_pyodide_version) + pyodide_version = cibw_pyodide_version + with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): if pyodide_root.exists(): return str(pyodide_root) From 735d5bb100705a6225ae90e959a2b031750ec55b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 22:58:38 +0530 Subject: [PATCH 015/109] Test wheel builds with Pyodide 0.26.2 --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 54e950e80..68a92cabf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -194,6 +194,7 @@ jobs: output-dir: wheelhouse env: CIBW_PLATFORM: pyodide + CIBW_PYODIDE_VERSION: 0.26.2 - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' run: | From b8ac6c015fff7cf3312d57683874eda238dd3f5d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 23:09:51 +0530 Subject: [PATCH 016/109] Add correct Pyodide version to identifier temp dir --- cibuildwheel/pyodide.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 50e0be661..f019c0202 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -268,7 +268,9 @@ def build(options: Options, tmp_path: Path) -> None: log.build_start(config.identifier) # Include both the identifier and the Pyodide version in the temp directory name - identifier_tmp_dir = tmp_path / f"{config.identifier}_{config.pyodide_version}" + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", config.pyodide_version) + identifier_tmp_dir = tmp_path / f"{config.identifier}_{cibw_pyodide_version}" + built_wheel_dir = identifier_tmp_dir / "built_wheel" repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" identifier_tmp_dir.mkdir() From 7796311dc62ec42461534404cb9dbffc5b77a05a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 23:19:59 +0530 Subject: [PATCH 017/109] Don't pre-call Pyodide xbuildenv search --- cibuildwheel/pyodide.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index f019c0202..3d3778305 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -107,19 +107,19 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" ) - # Validate that the requested xbuildenv version is compatible with the pyodide-build version. - # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, - # which will always be compatible. Hence, this function really only checks for the case where the - # version is supplied manually through CIBW_PYODIDE_VERSION environment variable. - cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION") - if cibw_pyodide_version: - validate_xbuildenv_version(pyodide_build_version, cibw_pyodide_version) - pyodide_version = cibw_pyodide_version - with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): if pyodide_root.exists(): return str(pyodide_root) + # Validate that the requested xbuildenv version is compatible with the pyodide-build version. + # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, + # which will always be compatible. Hence, this function really only checks for the case where the + # version is supplied manually through CIBW_PYODIDE_VERSION environment variable. + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION") + if cibw_pyodide_version: + validate_xbuildenv_version(pyodide_build_version, cibw_pyodide_version) + pyodide_version = cibw_pyodide_version + # We don't want to mutate env but we need to delete any existing # PYODIDE_ROOT so copy it first. env = dict(env) From 97e22c882534dec8ee686e51b2813735e16d9ee9 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 24 Sep 2024 23:20:31 +0530 Subject: [PATCH 018/109] Fetch just the stable Pyodide versions --- cibuildwheel/pyodide.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 3d3778305..68b3a529e 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -83,7 +83,12 @@ def retrieve_compatible_xbuildenvs() -> list[str]: env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] ] - return compatible_xbuildenvs + # Fetch just the "stable" versions + compatible_xbuildenvs_filtered = [ + version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") + ] + + return compatible_xbuildenvs_filtered def validate_xbuildenv_version(pyodide_build_version: str, xbuildenv_version: str) -> None: From d30eb6aa882ff0ea3d3061fb8f2193bf4b5f6097 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:04:09 +0530 Subject: [PATCH 019/109] Refactor search + validation + install into one step --- cibuildwheel/pyodide.py | 61 ++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 68b3a529e..4c9f24ba2 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -67,8 +67,20 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path -def retrieve_compatible_xbuildenvs() -> list[str]: - """Search for compatible xbuildenvs for the current pyodide-build version.""" +def search_and_install_xbuildenv( + env: dict[str, str], pyodide_build_version: str, pyodide_version: str +) -> str: + """Searches, validates, and installs a particular Pyodide xbuildenv version.""" + # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are + # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall + # specify the pyodide-build version in the root path, which will set up the xbuildenv for + # the requested Pyodide version. + pyodide_root = ( + CIBW_CACHE_PATH + / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" + ) + + # Search for compatible xbuildenvs for the current pyodide-build version xbuildenvs = call( "pyodide", "xbuildenv", @@ -77,54 +89,33 @@ def retrieve_compatible_xbuildenvs() -> list[str]: "--all", capture_stdout=True, ).strip() - xbuildenvs_dict = json.loads(xbuildenvs) compatible_xbuildenvs = [ env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] ] - # Fetch just the "stable" versions compatible_xbuildenvs_filtered = [ version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") ] - return compatible_xbuildenvs_filtered - - -def validate_xbuildenv_version(pyodide_build_version: str, xbuildenv_version: str) -> None: - """Validate that the requested xbuildenv version is compatible with the pyodide-build version.""" - xbuildenvs = retrieve_compatible_xbuildenvs() - if xbuildenv_version not in xbuildenvs: + # Now, validate that the requested xbuildenv version is compatible with the pyodide-build version. + # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, + # which will always be compatible. Hence, this condition really only checks for the case where the + # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable. + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version) + if cibw_pyodide_version not in compatible_xbuildenvs_filtered: msg = ( - f"The xbuildenv version {xbuildenv_version} is not compatible with the pyodide-build version {pyodide_build_version}." - f" The compatible versions are: {xbuildenvs}. Please use the 'pyodide xbuildenv search' command to find the compatible versions." + f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" + f" version {pyodide_build_version}. The compatible versions available to download are:" + f" {compatible_xbuildenvs_filtered}. Please use the 'pyodide xbuildenv search' command to" + f" find the compatible versions for {pyodide_build_version}" ) raise errors.FatalError(msg) - -def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: - # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are - # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall - # specify the pyodide-build version, which will set up the xbuildenv for the requested - # Pyodide version. - pyodide_root = ( - CIBW_CACHE_PATH - / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" - ) - with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): if pyodide_root.exists(): return str(pyodide_root) - # Validate that the requested xbuildenv version is compatible with the pyodide-build version. - # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, - # which will always be compatible. Hence, this function really only checks for the case where the - # version is supplied manually through CIBW_PYODIDE_VERSION environment variable. - cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION") - if cibw_pyodide_version: - validate_xbuildenv_version(pyodide_build_version, cibw_pyodide_version) - pyodide_version = cibw_pyodide_version - # We don't want to mutate env but we need to delete any existing # PYODIDE_ROOT so copy it first. env = dict(env) @@ -133,7 +124,7 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v "pyodide", "xbuildenv", "install", - pyodide_version, + cibw_pyodide_version, env=env, cwd=CIBW_CACHE_PATH, ) @@ -222,7 +213,7 @@ def setup_python( env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) log.step(f"Installing Pyodide xbuildenv version: {python_configuration.pyodide_version} ...") - env["PYODIDE_ROOT"] = install_xbuildenv( + env["PYODIDE_ROOT"] = search_and_install_xbuildenv( env, python_configuration.pyodide_build_version, python_configuration.pyodide_version ) From ce2a3f08b9725c7ddebea67f86a6efbe72124f6d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:20:46 +0530 Subject: [PATCH 020/109] Move all of it under a lock --- cibuildwheel/pyodide.py | 64 ++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 4c9f24ba2..ff46e5fd4 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -80,42 +80,42 @@ def search_and_install_xbuildenv( / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" ) - # Search for compatible xbuildenvs for the current pyodide-build version - xbuildenvs = call( - "pyodide", - "xbuildenv", - "search", - "--json", - "--all", - capture_stdout=True, - ).strip() - xbuildenvs_dict = json.loads(xbuildenvs) - compatible_xbuildenvs = [ - env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] - ] - # Fetch just the "stable" versions - compatible_xbuildenvs_filtered = [ - version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") - ] - - # Now, validate that the requested xbuildenv version is compatible with the pyodide-build version. - # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, - # which will always be compatible. Hence, this condition really only checks for the case where the - # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable. - cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version) - if cibw_pyodide_version not in compatible_xbuildenvs_filtered: - msg = ( - f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" - f" version {pyodide_build_version}. The compatible versions available to download are:" - f" {compatible_xbuildenvs_filtered}. Please use the 'pyodide xbuildenv search' command to" - f" find the compatible versions for {pyodide_build_version}" - ) - raise errors.FatalError(msg) - with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): if pyodide_root.exists(): return str(pyodide_root) + # Search for compatible xbuildenvs for the current pyodide-build version + xbuildenvs = call( + "pyodide", + "xbuildenv", + "search", + "--json", + "--all", + capture_stdout=True, + ).strip() + xbuildenvs_dict = json.loads(xbuildenvs) + compatible_xbuildenvs = [ + env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] + ] + # Fetch just the "stable" versions + compatible_xbuildenvs_filtered = [ + version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") + ] + + # Now, validate that the requested xbuildenv version is compatible with the pyodide-build version. + # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, + # which will always be compatible. Hence, this condition really only checks for the case where the + # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable. + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version) + if cibw_pyodide_version not in compatible_xbuildenvs_filtered: + msg = ( + f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" + f" version {pyodide_build_version}. The compatible versions available to download are:" + f" {compatible_xbuildenvs_filtered}. Please use the 'pyodide xbuildenv search' command to" + f" find the compatible versions for {pyodide_build_version}" + ) + raise errors.FatalError(msg) + # We don't want to mutate env but we need to delete any existing # PYODIDE_ROOT so copy it first. env = dict(env) From 13fbf66873c7e1a270d03da2b1824ff70b4f0eb4 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:28:48 +0530 Subject: [PATCH 021/109] Reorder xbuildenv installation --- cibuildwheel/pyodide.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index ff46e5fd4..353d9b71b 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -84,7 +84,12 @@ def search_and_install_xbuildenv( if pyodide_root.exists(): return str(pyodide_root) - # Search for compatible xbuildenvs for the current pyodide-build version + # We don't want to mutate env but we need to delete any existing + # PYODIDE_ROOT so copy it first. + env = dict(env) + env.pop("PYODIDE_ROOT", None) + + # 1. Search for compatible xbuildenvs for the current pyodide-build version xbuildenvs = call( "pyodide", "xbuildenv", @@ -102,7 +107,7 @@ def search_and_install_xbuildenv( version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") ] - # Now, validate that the requested xbuildenv version is compatible with the pyodide-build version. + # 2. Now, validate that the requested xbuildenv version is compatible with the pyodide-build version. # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, # which will always be compatible. Hence, this condition really only checks for the case where the # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable. @@ -116,10 +121,7 @@ def search_and_install_xbuildenv( ) raise errors.FatalError(msg) - # We don't want to mutate env but we need to delete any existing - # PYODIDE_ROOT so copy it first. - env = dict(env) - env.pop("PYODIDE_ROOT", None) + # 3. Install the xbuildenv call( "pyodide", "xbuildenv", From 14ec0712667dcca54e18efb83012cb7bd3ee0564 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:34:49 +0530 Subject: [PATCH 022/109] Add env and cwd to xbuildenv search call --- cibuildwheel/pyodide.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 353d9b71b..04dcfe732 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -96,6 +96,8 @@ def search_and_install_xbuildenv( "search", "--json", "--all", + env=env, + cwd=CIBW_CACHE_PATH, capture_stdout=True, ).strip() xbuildenvs_dict = json.loads(xbuildenvs) From aae64bbc1e2631156ee03cf21fd259f09c54e63f Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 00:39:33 +0530 Subject: [PATCH 023/109] Temporarily lower to 0.26.2 target --- cibuildwheel/resources/build-platforms.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 15a711f54..f2ca53ce5 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -172,5 +172,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.27.0a2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, ] From 6956121375229028d0d10b74c2689153097eba89 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 01:34:00 +0530 Subject: [PATCH 024/109] Separate out search, validate, install; again --- cibuildwheel/pyodide.py | 97 +++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 04dcfe732..862df5603 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -67,10 +67,49 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path -def search_and_install_xbuildenv( - env: dict[str, str], pyodide_build_version: str, pyodide_version: str -) -> str: - """Searches, validates, and installs a particular Pyodide xbuildenv version.""" +def search_xbuildenv() -> list[str]: + """Searches for the compatible xbuildenvs for the current pyodide-build version""" + xbuildenvs = call( + "pyodide", + "xbuildenv", + "search", + "--json", + "--all", + capture_stdout=True, + ).strip() + xbuildenvs_dict = json.loads(xbuildenvs) + compatible_xbuildenvs = [ + env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] + ] + # Fetch just the "stable" versions + compatible_xbuildenvs_filtered = [ + version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") + ] + + return compatible_xbuildenvs_filtered + + +def validate_xbuildenv(pyodide_version: str, pyodide_build_version: str) -> None: + """Validate the Pyodide version if set manually for the current pyodide-build version""" + pyodide_versions = search_xbuildenv() + + # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, + # which will always be compatible. Hence, this condition really checks only for the case where the + # version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an + # error as appropriate. + cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version) + if cibw_pyodide_version not in pyodide_versions: + msg = ( + f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" + f" version {pyodide_build_version}. The compatible versions available to download are:" + f" {pyodide_versions}. Please use the 'pyodide xbuildenv search' command to" + f" find the compatible versions for {pyodide_build_version}" + ) + raise errors.FatalError(msg) + + +def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: + """Installs a particular Pyodide xbuildenv version.""" # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall # specify the pyodide-build version in the root path, which will set up the xbuildenv for @@ -89,46 +128,12 @@ def search_and_install_xbuildenv( env = dict(env) env.pop("PYODIDE_ROOT", None) - # 1. Search for compatible xbuildenvs for the current pyodide-build version - xbuildenvs = call( - "pyodide", - "xbuildenv", - "search", - "--json", - "--all", - env=env, - cwd=CIBW_CACHE_PATH, - capture_stdout=True, - ).strip() - xbuildenvs_dict = json.loads(xbuildenvs) - compatible_xbuildenvs = [ - env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] - ] - # Fetch just the "stable" versions - compatible_xbuildenvs_filtered = [ - version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") - ] - - # 2. Now, validate that the requested xbuildenv version is compatible with the pyodide-build version. - # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, - # which will always be compatible. Hence, this condition really only checks for the case where the - # version is supplied manually through the CIBW_PYODIDE_VERSION environment variable. - cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version) - if cibw_pyodide_version not in compatible_xbuildenvs_filtered: - msg = ( - f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" - f" version {pyodide_build_version}. The compatible versions available to download are:" - f" {compatible_xbuildenvs_filtered}. Please use the 'pyodide xbuildenv search' command to" - f" find the compatible versions for {pyodide_build_version}" - ) - raise errors.FatalError(msg) - # 3. Install the xbuildenv call( "pyodide", "xbuildenv", "install", - cibw_pyodide_version, + pyodide_version, env=env, cwd=CIBW_CACHE_PATH, ) @@ -216,9 +221,17 @@ def setup_python( env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) - log.step(f"Installing Pyodide xbuildenv version: {python_configuration.pyodide_version} ...") - env["PYODIDE_ROOT"] = search_and_install_xbuildenv( - env, python_configuration.pyodide_build_version, python_configuration.pyodide_version + # Allow overriding the xbuildenv version with an environment variable. This allows + # testing new Pyodide xbuildenv versions before they are officially released, or for + # using a different Pyodide version other than the one that is listed to be compatible + # with the pyodide-build version in the build-platforms.toml file. + cibw_pyodide_version = os.environ.get( + "CIBW_PYODIDE_VERSION", python_configuration.pyodide_version + ) + log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...") + validate_xbuildenv(cibw_pyodide_version, cibw_pyodide_version) + env["PYODIDE_ROOT"] = install_xbuildenv( + env, python_configuration.pyodide_build_version, cibw_pyodide_version ) return env From e5d8443586839839109c3972c5ee4bb05462b332 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 01:38:46 +0530 Subject: [PATCH 025/109] Run xbuildenv search in `CIBW_CACHE_PATH` --- cibuildwheel/pyodide.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 862df5603..590856176 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -75,6 +75,7 @@ def search_xbuildenv() -> list[str]: "search", "--json", "--all", + cwd=CIBW_CACHE_PATH, capture_stdout=True, ).strip() xbuildenvs_dict = json.loads(xbuildenvs) From 95c368116aad3b32a87b33169dda7251e6f1a396 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 01:54:57 +0530 Subject: [PATCH 026/109] Remove prior `PYODIDE_ROOT` env vars, copy envs --- cibuildwheel/pyodide.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 590856176..079f2e0d8 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -69,12 +69,17 @@ def install_emscripten(tmp: Path, version: str) -> Path: def search_xbuildenv() -> list[str]: """Searches for the compatible xbuildenvs for the current pyodide-build version""" + + env = dict(os.environ) + env.pop("PYODIDE_ROOT", None) + xbuildenvs = call( "pyodide", "xbuildenv", "search", "--json", "--all", + env=env, cwd=CIBW_CACHE_PATH, capture_stdout=True, ).strip() @@ -86,6 +91,7 @@ def search_xbuildenv() -> list[str]: compatible_xbuildenvs_filtered = [ version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") ] + # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions return compatible_xbuildenvs_filtered @@ -110,7 +116,7 @@ def validate_xbuildenv(pyodide_version: str, pyodide_build_version: str) -> None def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: - """Installs a particular Pyodide xbuildenv version.""" + """Install a particular Pyodide xbuildenv version and set a path to the Pyodide root.""" # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall # specify the pyodide-build version in the root path, which will set up the xbuildenv for From 09af46f14c9dede60160727878f74f7fb292c029 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 02:08:03 +0530 Subject: [PATCH 027/109] Validate doesn't need to depend on searching --- cibuildwheel/pyodide.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 079f2e0d8..b0ac81dcb 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -67,12 +67,8 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path -def search_xbuildenv() -> list[str]: +def search_xbuildenv(env: dict[str, str]) -> list[str]: """Searches for the compatible xbuildenvs for the current pyodide-build version""" - - env = dict(os.environ) - env.pop("PYODIDE_ROOT", None) - xbuildenvs = call( "pyodide", "xbuildenv", @@ -96,20 +92,21 @@ def search_xbuildenv() -> list[str]: return compatible_xbuildenvs_filtered -def validate_xbuildenv(pyodide_version: str, pyodide_build_version: str) -> None: - """Validate the Pyodide version if set manually for the current pyodide-build version""" - pyodide_versions = search_xbuildenv() +# The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, +# which will always be compatible. Hence, this condition really checks only for the case where the +# version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an +# error as appropriate. +def validate_xbuildenv( + cibw_pyodide_version: str, pyodide_build_version: str, compatible_versions: list[str] +) -> None: + """Validate the Pyodide version if set manually for the current pyodide-build version + against a list of compatible versions.""" - # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, - # which will always be compatible. Hence, this condition really checks only for the case where the - # version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an - # error as appropriate. - cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", pyodide_version) - if cibw_pyodide_version not in pyodide_versions: + if cibw_pyodide_version not in compatible_versions: msg = ( f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" f" version {pyodide_build_version}. The compatible versions available to download are:" - f" {pyodide_versions}. Please use the 'pyodide xbuildenv search' command to" + f" {compatible_versions}. Please use the 'pyodide xbuildenv search' command to" f" find the compatible versions for {pyodide_build_version}" ) raise errors.FatalError(msg) @@ -236,7 +233,12 @@ def setup_python( "CIBW_PYODIDE_VERSION", python_configuration.pyodide_version ) log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...") - validate_xbuildenv(cibw_pyodide_version, cibw_pyodide_version) + # Search for compatible xbuildenv versions + compatible_versions = search_xbuildenv(env) + # and then validate the xbuildenv version + validate_xbuildenv( + cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions + ) env["PYODIDE_ROOT"] = install_xbuildenv( env, python_configuration.pyodide_build_version, cibw_pyodide_version ) From 66999cbdc45e6734ff0d69fe3f8593cac4f69b47 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 02:08:47 +0530 Subject: [PATCH 028/109] Add file lock when searching xbuildenvs --- cibuildwheel/pyodide.py | 43 +++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index b0ac81dcb..bc463da04 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -69,27 +69,28 @@ def install_emscripten(tmp: Path, version: str) -> Path: def search_xbuildenv(env: dict[str, str]) -> list[str]: """Searches for the compatible xbuildenvs for the current pyodide-build version""" - xbuildenvs = call( - "pyodide", - "xbuildenv", - "search", - "--json", - "--all", - env=env, - cwd=CIBW_CACHE_PATH, - capture_stdout=True, - ).strip() - xbuildenvs_dict = json.loads(xbuildenvs) - compatible_xbuildenvs = [ - env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] - ] - # Fetch just the "stable" versions - compatible_xbuildenvs_filtered = [ - version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") - ] - # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions - - return compatible_xbuildenvs_filtered + with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): + xbuildenvs = call( + "pyodide", + "xbuildenv", + "search", + "--json", + "--all", + env=env, + cwd=CIBW_CACHE_PATH, + capture_stdout=True, + ).strip() + xbuildenvs_dict = json.loads(xbuildenvs) + compatible_xbuildenvs = [ + env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] + ] + # Fetch just the "stable" versions + compatible_xbuildenvs_filtered = [ + version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") + ] + # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions + + return compatible_xbuildenvs_filtered # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, From 1af1e5dd8464bd27dfff8134b4d73cb5b9a0b704 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Wed, 25 Sep 2024 03:06:25 +0530 Subject: [PATCH 029/109] Test the original version: 0.26.1 --- .github/workflows/test.yml | 1 - cibuildwheel/resources/build-platforms.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 68a92cabf..54e950e80 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -194,7 +194,6 @@ jobs: output-dir: wheelhouse env: CIBW_PLATFORM: pyodide - CIBW_PYODIDE_VERSION: 0.26.2 - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' run: | diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index f2ca53ce5..9c4cc2a4c 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -172,5 +172,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.2", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, ] From d3445483bc5ffde485485afb8f0980462084cb7a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 22 Oct 2024 17:22:32 +0530 Subject: [PATCH 030/109] Update Pyodide constraints --- cibuildwheel/resources/constraints-pyodide312.txt | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 394806f20..f575ad707 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -19,8 +19,6 @@ charset-normalizer==3.4.0 # via requests click==8.1.7 # via typer -cloudpickle==3.1.0 - # via loky cmake==3.30.5 # via pyodide-build distlib==0.3.9 @@ -96,8 +94,6 @@ typer==0.12.5 # auditwheel-emscripten # pyodide-build # pyodide-cli -types-requests==2.32.0.20241016 - # via pyodide-build typing-extensions==4.12.2 # via # pydantic @@ -106,9 +102,7 @@ typing-extensions==4.12.2 unearth==0.17.2 # via pyodide-build urllib3==2.2.3 - # via - # requests - # types-requests + # via requests virtualenv==20.27.0 # via # build From 738ed1f5e5a18b7d5a67c19d43820eeef789dd1d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:03:58 +0530 Subject: [PATCH 031/109] Update constraints for `pyodide-build` 0.29.0 again --- cibuildwheel/resources/constraints-pyodide312.txt | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 1920c59da..a0fcc3bee 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -19,8 +19,6 @@ charset-normalizer==3.4.0 # via requests click==8.1.7 # via typer -cloudpickle==3.1.0 - # via loky cmake==3.31.0.1 # via pyodide-build distlib==0.3.9 @@ -54,11 +52,11 @@ pip==24.3.1 # via -r .nox/update_constraints/tmp/constraints-pyodide.in platformdirs==4.3.6 # via virtualenv -pydantic==2.9.2 +pydantic==2.10.0 # via # pyodide-build # pyodide-lock -pydantic-core==2.23.4 +pydantic-core==2.27.0 # via pydantic pygments==2.18.0 # via rich @@ -91,7 +89,7 @@ sniffio==1.3.1 # via # anyio # httpx -typer==0.13.0 +typer==0.13.1 # via # auditwheel-emscripten # pyodide-build @@ -104,9 +102,7 @@ typing-extensions==4.12.2 unearth==0.17.2 # via pyodide-build urllib3==2.2.3 - # via - # requests - # types-requests + # via requests virtualenv==20.27.1 # via # build From 41e466a487b3d411b6dbd4847ad82fab404fb94e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 14:12:22 +0530 Subject: [PATCH 032/109] =?UTF-8?q?Bump=20Pyodide=20from=20version=200.26.?= =?UTF-8?q?1=20=E2=9E=A1=EF=B8=8F=20version=200.26.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cibuildwheel/resources/build-platforms.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 8cfa436e6..6e2d296b2 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -190,5 +190,5 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.1", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.4", pyodide_build_version = "0.29.0", emscripten_version = "3.1.58", node_version = "v20" }, ] From 2a66ddd0a8e6ed531c5e033a2992ad8435b627ee Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:15:44 +0530 Subject: [PATCH 033/109] Add note on compatibility for macOS + other archs --- docs/options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index 327bfe1f8..3970c3cb2 100644 --- a/docs/options.md +++ b/docs/options.md @@ -255,7 +255,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. -- For `pyodide` you need to be on an x86-64 linux runner and `python3.12` must be available in `PATH`. +- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may work, too. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. From dff72caa48423f85c50d30483c2e13421a6436e9 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:15:58 +0530 Subject: [PATCH 034/109] Note Pyodide version for Pyodide identifier --- docs/options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index 3970c3cb2..4ce75774c 100644 --- a/docs/options.md +++ b/docs/options.md @@ -321,7 +321,7 @@ If you set the value lower, cibuildwheel will cap it to the lowest supported val Windows arm64 platform support is experimental. For an experimental WebAssembly build with `--platform pyodide`, -`cp312-pyodide_wasm32` is the only platform identifier. +`cp312-pyodide_wasm32` is the only platform identifier, corresponding to [Pyodide version `0.26.4`](https://github.com/pyodide/pyodide/releases/tag/0.26.4). See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) for past end-of-life versions of Python, and PyPy2.7. From 9a78845d3d7d8852f495a6db66f369ac54f124d4 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:16:14 +0530 Subject: [PATCH 035/109] Docs about `CIBW_PYODIDE_VERSION` --- docs/setup.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/setup.md b/docs/setup.md index 32e0d01c5..e900eef4b 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -113,7 +113,14 @@ Pre-requisite: you need to have a matching host version of Python (unlike all other cibuildwheel platforms). Linux host highly recommended; macOS hosts may work (e.g. invoking `pytest` directly in [`CIBW_TEST_COMMAND`](options.md#test-command) is [currently failing](https://github.com/pyodide/pyodide/issues/4802)) and Windows hosts will not work. -You must target pyodide with `--platform pyodide` (or use `--only` on the identifier). +You must target Pyodide with `--platform pyodide` (or use `--only` on the identifier). + +!!!tip + + It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` environment variable to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json) or can be + referenced using the `pyodide xbuildenv search` command. + + This option is **not available** in the `pyproject.toml` config. ## Configure a CI service From e551021a1bf71d8966baa6ace43adf162a6f7e23 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:47:09 +0530 Subject: [PATCH 036/109] Don't fetch just the stable versions --- cibuildwheel/pyodide.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 6c547124d..843e5339f 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -81,16 +81,11 @@ def search_xbuildenv(env: dict[str, str]) -> list[str]: capture_stdout=True, ).strip() xbuildenvs_dict = json.loads(xbuildenvs) - compatible_xbuildenvs = [ + compatible_xbuildenv_versions = [ env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] ] - # Fetch just the "stable" versions - compatible_xbuildenvs_filtered = [ - version for version in compatible_xbuildenvs if not any(_ in version for _ in "abc") - ] - # TODO: possibly remove that? Since this won't allow testing the unstable/dev versions - return compatible_xbuildenvs_filtered + return compatible_xbuildenv_versions # The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, From 1fe8a04d0a7c72b55697685dd358d47756aac54d Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:26:16 +0530 Subject: [PATCH 037/109] Discard a variable that's not used later --- cibuildwheel/pyodide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 843e5339f..44c5ccbb9 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -82,7 +82,7 @@ def search_xbuildenv(env: dict[str, str]) -> list[str]: ).strip() xbuildenvs_dict = json.loads(xbuildenvs) compatible_xbuildenv_versions = [ - env["version"] for env in xbuildenvs_dict["environments"] if env["compatible"] + _["version"] for _ in xbuildenvs_dict["environments"] if _["compatible"] ] return compatible_xbuildenv_versions From 8a8177c646f365bb8654f9f5560446acadf246bd Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:29:02 +0530 Subject: [PATCH 038/109] =?UTF-8?q?Rename=20`search=5Fxbuildenv`=20?= =?UTF-8?q?=E2=9E=A1=EF=B8=8F=20`get=5Fxbuildenv=5Fversions`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cibuildwheel/pyodide.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 44c5ccbb9..b6cb1a427 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -67,7 +67,7 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path -def search_xbuildenv(env: dict[str, str]) -> list[str]: +def get_xbuildenv_versions(env: dict[str, str]) -> list[str]: """Searches for the compatible xbuildenvs for the current pyodide-build version""" with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): xbuildenvs = call( @@ -230,7 +230,7 @@ def setup_python( ) log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...") # Search for compatible xbuildenv versions - compatible_versions = search_xbuildenv(env) + compatible_versions = get_xbuildenv_versions(env) # and then validate the xbuildenv version validate_xbuildenv( cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions From fffb7050aa3eb15147b9409f2ca30f6f01092a8a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:29:32 +0530 Subject: [PATCH 039/109] =?UTF-8?q?`validate=5Fxbuildenv`=20=E2=9E=A1?= =?UTF-8?q?=EF=B8=8F=20`validate=5Fxbuildenv=5Fversion`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cibuildwheel/pyodide.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index b6cb1a427..2f403d499 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -92,7 +92,7 @@ def get_xbuildenv_versions(env: dict[str, str]) -> list[str]: # which will always be compatible. Hence, this condition really checks only for the case where the # version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an # error as appropriate. -def validate_xbuildenv( +def validate_xbuildenv_version( cibw_pyodide_version: str, pyodide_build_version: str, compatible_versions: list[str] ) -> None: """Validate the Pyodide version if set manually for the current pyodide-build version @@ -232,7 +232,7 @@ def setup_python( # Search for compatible xbuildenv versions compatible_versions = get_xbuildenv_versions(env) # and then validate the xbuildenv version - validate_xbuildenv( + validate_xbuildenv_version( cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions ) env["PYODIDE_ROOT"] = install_xbuildenv( From 0df3c45f28761a16f56bee9441805f8e8f2ab23b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:36:58 +0530 Subject: [PATCH 040/109] Replace ordered comment, add newline --- cibuildwheel/pyodide.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 2f403d499..6fb4447ae 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -101,9 +101,9 @@ def validate_xbuildenv_version( if cibw_pyodide_version not in compatible_versions: msg = ( f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" - f" version {pyodide_build_version}. The compatible versions available to download are:" - f" {compatible_versions}. Please use the 'pyodide xbuildenv search' command to" - f" find the compatible versions for {pyodide_build_version}" + f" version {pyodide_build_version}. The compatible versions available to download are:\n" + f"{compatible_versions}. Please use the 'pyodide xbuildenv search' command to" + f" find the compatible versions for {pyodide_build_version}." ) raise errors.FatalError(msg) @@ -128,7 +128,7 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v env = dict(env) env.pop("PYODIDE_ROOT", None) - # 3. Install the xbuildenv + # Install the xbuildenv call( "pyodide", "xbuildenv", From 057e542fa3a670dd4e92773e0e33a12b806dbc6a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:37:53 +0530 Subject: [PATCH 041/109] Replace sentence on macOS support Co-Authored-By: Hood Chatham --- docs/options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index 4ce75774c..8281836ef 100644 --- a/docs/options.md +++ b/docs/options.md @@ -255,7 +255,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. -- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may work, too. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information. +- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts work, though there are known bugs. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. From 29b5effd8c586d5a2705c43304ba4c4922dbc87c Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 18:40:31 +0530 Subject: [PATCH 042/109] =?UTF-8?q?Capitalise:=20"pyodide"=20=E2=9E=A1?= =?UTF-8?q?=EF=B8=8F=20"Pyodide"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Hood Chatham --- test/test_emscripten.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_emscripten.py b/test/test_emscripten.py index aa7750bc5..7a8fd9953 100644 --- a/test/test_emscripten.py +++ b/test/test_emscripten.py @@ -20,7 +20,7 @@ def check_node(): # cibuildwheel adds a pinned node version to the PATH - # check it's in the PATH then, check it's the one that runs pyodide + # check it's in the PATH then, check it's the one that runs Pyodide cibw_cache_path = Path(sys.argv[1]).resolve(strict=True) # find the node executable in PATH node = shutil.which("node") From 321e0dee3b7facc42fa8e913c56a96a810c9817a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 20:21:26 +0530 Subject: [PATCH 043/109] =?UTF-8?q?"work"=20=E2=9E=A1=EF=B8=8F=20"may=20su?= =?UTF-8?q?cceed"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Hood Chatham --- docs/options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index 8281836ef..d8cff279c 100644 --- a/docs/options.md +++ b/docs/options.md @@ -255,7 +255,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. -- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts work, though there are known bugs. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information. +- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may succeed, though there are known bugs. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. From 92babdb76184f1b5b29b310ea223c7cbef49ef36 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 20:28:00 +0530 Subject: [PATCH 044/109] Add another job to test a custom Pyodide version --- .github/workflows/test.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3757be634..45ba25b09 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -171,9 +171,13 @@ jobs: run: uv run pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py test-pyodide: - name: Test cibuildwheel building pyodide wheels + name: Test cibuildwheel building Pyodide wheels (${{ matrix.pyodide-version }} version) needs: lint runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + pyodide-version: ["default", "custom"] timeout-minutes: 180 steps: - uses: actions/checkout@v4 @@ -191,12 +195,26 @@ jobs: uv run -m test.test_projects test.test_0_basic.basic_project sample_proj - name: Run a sample build (GitHub Action) + if: matrix.pyodide-version == 'default' + uses: ./ + with: + package-dir: sample_proj + output-dir: wheelhouse + env: + CIBW_PLATFORM: pyodide + + - name: Run a sample build (GitHub Action) for an overridden Pyodide version + if: matrix.pyodide-version == 'custom' uses: ./ with: package-dir: sample_proj output-dir: wheelhouse + # In case this breaks at any point in time, switch to using the latest version + # available or any other version that is not the same as the default one set + # in cibuildwheel/resources/build-platforms.toml. env: CIBW_PLATFORM: pyodide + CIBW_PYODIDE_VERSION: "v0.27.0a2" - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' run: | From d73f71f8c812971d98205e76dae40325157d7baa Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 21 Nov 2024 21:32:02 +0530 Subject: [PATCH 045/109] Handle "v"-prefixed + non-prefixed versions --- .github/workflows/test.yml | 2 +- cibuildwheel/pyodide.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 45ba25b09..6320821a7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -214,7 +214,7 @@ jobs: # in cibuildwheel/resources/build-platforms.toml. env: CIBW_PLATFORM: pyodide - CIBW_PYODIDE_VERSION: "v0.27.0a2" + CIBW_PYODIDE_VERSION: "0.27.0a2" - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' run: | diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 6fb4447ae..b8f688094 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -103,7 +103,7 @@ def validate_xbuildenv_version( f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" f" version {pyodide_build_version}. The compatible versions available to download are:\n" f"{compatible_versions}. Please use the 'pyodide xbuildenv search' command to" - f" find the compatible versions for {pyodide_build_version}." + f" find the compatible versions for 'pyodide-build' {pyodide_build_version}." ) raise errors.FatalError(msg) @@ -228,6 +228,8 @@ def setup_python( cibw_pyodide_version = os.environ.get( "CIBW_PYODIDE_VERSION", python_configuration.pyodide_version ) + # If there's a "v" prefix, remove it: both would be equally valid + cibw_pyodide_version = cibw_pyodide_version.lstrip("v") log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...") # Search for compatible xbuildenv versions compatible_versions = get_xbuildenv_versions(env) From 1b4f911c1118eb60627e95bbc59f97093293e9dc Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 21 Mar 2025 18:26:30 +0000 Subject: [PATCH 046/109] Convert to a proper toml-able option, and remove some hardcoded versions This removes the enscripten and pyodide-build version specs from pyproject.toml - pyodide-build is spec'd in the constraints file, and the emscripten version can be read from the pyodide-build output. --- cibuildwheel/options.py | 4 + cibuildwheel/pyodide.py | 179 ++++++++++++-------- cibuildwheel/resources/build-platforms.toml | 2 +- cibuildwheel/resources/defaults.toml | 2 + cibuildwheel/util/helpers.py | 15 ++ unit_test/utils_test.py | 32 ++++ 6 files changed, 161 insertions(+), 73 deletions(-) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index fa7605343..287f1d02f 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -108,6 +108,7 @@ class BuildOptions: build_frontend: BuildFrontendConfig | None config_settings: str container_engine: OCIContainerEngineConfig + pyodide_version: str | None @property def package_dir(self) -> Path: @@ -825,6 +826,8 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: msg = f"Failed to parse container config. {e}" raise errors.ConfigurationError(msg) from e + pyodide_version = self.reader.get("pyodide-version", env_plat=False) + return BuildOptions( globals=self.globals, test_command=test_command, @@ -844,6 +847,7 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: build_frontend=build_frontend, config_settings=config_settings, container_engine=container_engine, + pyodide_version=pyodide_version or None, ) def check_for_invalid_configuration(self, identifiers: Iterable[str]) -> None: diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 8598a309e..97918e38e 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -4,11 +4,12 @@ import shutil import sys import tomllib +import typing from collections.abc import Sequence, Set from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory -from typing import Final +from typing import Final, TypedDict from filelock import FileLock @@ -30,7 +31,7 @@ extract_zip, move_file, ) -from .util.helpers import prepare_command +from .util.helpers import prepare_command, unwrap, unwrap_preserving_paragraphs from .util.packaging import combine_constraints, find_compatible_wheel, get_pip_version from .venv import virtualenv @@ -41,12 +42,23 @@ class PythonConfiguration: version: str identifier: str - pyodide_version: str - pyodide_build_version: str - emscripten_version: str + default_pyodide_version: str node_version: str +class PyodideXBuildEnvInfoVersionRange(TypedDict): + min: str | None + max: str | None + + +class PyodideXBuildEnvInfo(TypedDict): + version: str + python: str + emscripten: str + pyodide_build: PyodideXBuildEnvInfoVersionRange + compatible: bool + + @functools.cache def ensure_node(major_version: str) -> Path: with resources.NODEJS.open("rb") as f: @@ -79,8 +91,6 @@ def ensure_node(major_version: str) -> Path: def install_emscripten(tmp: Path, version: str) -> Path: - # We don't need to match the emsdk version to the version we install, but - # we do for stability url = f"https://github.com/emscripten-core/emsdk/archive/refs/tags/{version}.zip" installation_path = CIBW_CACHE_PATH / f"emsdk-{version}" emsdk_path = installation_path / f"emsdk-{version}/emsdk" @@ -98,44 +108,70 @@ def install_emscripten(tmp: Path, version: str) -> Path: return emcc_path -def get_xbuildenv_versions(env: dict[str, str]) -> list[str]: - """Searches for the compatible xbuildenvs for the current pyodide-build version""" - with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): - xbuildenvs = call( - "pyodide", - "xbuildenv", - "search", - "--json", - "--all", - env=env, - cwd=CIBW_CACHE_PATH, - capture_stdout=True, - ).strip() - xbuildenvs_dict = json.loads(xbuildenvs) - compatible_xbuildenv_versions = [ - _["version"] for _ in xbuildenvs_dict["environments"] if _["compatible"] - ] - - return compatible_xbuildenv_versions - - -# The xbuildenv version is brought in sync with the pyodide-build version in build-platforms.toml, -# which will always be compatible. Hence, this condition really checks only for the case where the -# version is supplied manually through a CIBW_PYODIDE_VERSION environment variable and raises an -# error as appropriate. -def validate_xbuildenv_version( - cibw_pyodide_version: str, pyodide_build_version: str, compatible_versions: list[str] +def get_all_xbuildenv_version_info(env: dict[str, str]) -> list[PyodideXBuildEnvInfo]: + xbuildenvs_info_str = call( + "pyodide", + "xbuildenv", + "search", + "--json", + "--all", + env=env, + cwd=CIBW_CACHE_PATH, + capture_stdout=True, + ).strip() + + xbuildenvs_info = json.loads(xbuildenvs_info_str) + + if "environments" not in xbuildenvs_info: + msg = f"Invalid xbuildenvs info, got {xbuildenvs_info}" + raise ValueError(msg) + + return typing.cast(list[PyodideXBuildEnvInfo], xbuildenvs_info["environments"]) + + +def get_xbuildenv_version_info( + env: dict[str, str], version: str, pyodide_build_version: str +) -> PyodideXBuildEnvInfo: + xbuildenvs_info = get_all_xbuildenv_version_info(env) + for xbuildenv_info in xbuildenvs_info: + if xbuildenv_info["version"] == version: + return xbuildenv_info + + msg = unwrap(f""" + Could not find pyodide xbuildenv version {version} in the available + versions as reported by pyodide-build v{pyodide_build_version}. + Available pyodide xbuild versions are: + {", ".join(e["version"] for e in xbuildenvs_info if e["compatible"])} + """) + raise errors.FatalError(msg) + + +# The xbuildenv version is brought in sync with the pyodide-build version in +# the constraints file, which will always be compatible with the version in +# build-platforms.toml. Hence, this condition really checks only for the case +# where the version is supplied manually through a CIBW_PYODIDE_VERSION +# environment variable and raises an error as appropriate. +def validate_pyodide_build_version( + xbuildenv_info: PyodideXBuildEnvInfo, pyodide_build_version: str ) -> None: - """Validate the Pyodide version if set manually for the current pyodide-build version - against a list of compatible versions.""" - - if cibw_pyodide_version not in compatible_versions: - msg = ( - f"The xbuildenv version {cibw_pyodide_version} is not compatible with the pyodide-build" - f" version {pyodide_build_version}. The compatible versions available to download are:\n" - f"{compatible_versions}. Please use the 'pyodide xbuildenv search' command to" - f" find the compatible versions for 'pyodide-build' {pyodide_build_version}." - ) + """ + Validate the Pyodide version is compatible with the installed + pyodide-build version. + """ + + pyodide_version = xbuildenv_info["version"] + + if not xbuildenv_info["compatible"]: + msg = unwrap_preserving_paragraphs(f""" + The Pyodide xbuildenv version {pyodide_version} is not compatible + with the pyodide-build version {pyodide_build_version}. Please use + the 'pyodide xbuildenv search --all' command to find a compatible + version. + + Set the pyodide-build version using the `dependency-constraints` + option, or set the Pyodide xbuildenv version using the + `pyodide-version` option. + """) raise errors.FatalError(msg) @@ -192,8 +228,10 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, + user_pyodide_version: str | None, ) -> dict[str, str]: base_python = get_base_python(python_configuration.identifier) + pyodide_version = user_pyodide_version or python_configuration.default_pyodide_version log.step("Setting up build environment...") venv_path = tmp / "venv" @@ -247,30 +285,28 @@ def setup_python( env=env, ) - log.step(f"Installing Emscripten version: {python_configuration.emscripten_version} ...") - emcc_path = install_emscripten(tmp, python_configuration.emscripten_version) + pyodide_build_version = call( + "python", + "-c", + "from importlib.metadata import version; print(version('pyodide-build'))", + env=env, + capture_stdout=True, + ).strip() + + xbuildenv_info = get_xbuildenv_version_info(env, pyodide_version, pyodide_build_version) + validate_pyodide_build_version( + xbuildenv_info=xbuildenv_info, + pyodide_build_version=pyodide_build_version, + ) + + enscripten_version = xbuildenv_info["emscripten"] + log.step(f"Installing Emscripten version: {enscripten_version} ...") + emcc_path = install_emscripten(tmp, enscripten_version) env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) - # Allow overriding the xbuildenv version with an environment variable. This allows - # testing new Pyodide xbuildenv versions before they are officially released, or for - # using a different Pyodide version other than the one that is listed to be compatible - # with the pyodide-build version in the build-platforms.toml file. - cibw_pyodide_version = os.environ.get( - "CIBW_PYODIDE_VERSION", python_configuration.pyodide_version - ) - # If there's a "v" prefix, remove it: both would be equally valid - cibw_pyodide_version = cibw_pyodide_version.lstrip("v") - log.step(f"Installing Pyodide xbuildenv version: {cibw_pyodide_version} ...") - # Search for compatible xbuildenv versions - compatible_versions = get_xbuildenv_versions(env) - # and then validate the xbuildenv version - validate_xbuildenv_version( - cibw_pyodide_version, python_configuration.pyodide_build_version, compatible_versions - ) - env["PYODIDE_ROOT"] = install_xbuildenv( - env, python_configuration.pyodide_build_version, cibw_pyodide_version - ) + log.step(f"Installing Pyodide xbuildenv version: {pyodide_version} ...") + env["PYODIDE_ROOT"] = install_xbuildenv(env, pyodide_build_version, pyodide_version) return env @@ -318,9 +354,7 @@ def build(options: Options, tmp_path: Path) -> None: log.build_start(config.identifier) - # Include both the identifier and the Pyodide version in the temp directory name - cibw_pyodide_version = os.environ.get("CIBW_PYODIDE_VERSION", config.pyodide_version) - identifier_tmp_dir = tmp_path / f"{config.identifier}_{cibw_pyodide_version}" + identifier_tmp_dir = tmp_path / config.identifier built_wheel_dir = identifier_tmp_dir / "built_wheel" repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel" @@ -336,10 +370,11 @@ def build(options: Options, tmp_path: Path) -> None: ) env = setup_python( - identifier_tmp_dir / "build", - config, - dependency_constraint_flags, - build_options.environment, + tmp=identifier_tmp_dir / "build", + python_configuration=config, + dependency_constraint_flags=dependency_constraint_flags, + environment=build_options.environment, + user_pyodide_version=build_options.pyodide_version, ) pip_version = get_pip_version(env) # The Pyodide command line runner mounts all directories in the host diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index b5fa071a2..5803ef98d 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -161,7 +161,7 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12", pyodide_version = "0.27.0", pyodide_build_version = "0.29.2", emscripten_version = "3.1.58", node_version = "v20" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.0", node_version = "v22" }, ] [ios] diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 6b65e6369..3a77055fa 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -25,6 +25,8 @@ test-groups = [] container-engine = "docker" +pyodide-version = "" + manylinux-x86_64-image = "manylinux2014" manylinux-i686-image = "manylinux2014" manylinux-aarch64-image = "manylinux2014" diff --git a/cibuildwheel/util/helpers.py b/cibuildwheel/util/helpers.py index dbef452c0..363fd3345 100644 --- a/cibuildwheel/util/helpers.py +++ b/cibuildwheel/util/helpers.py @@ -76,6 +76,21 @@ def unwrap(text: str) -> str: return re.sub(r"\s+", " ", text) +def unwrap_preserving_paragraphs(text: str) -> str: + """ + Unwraps multi-line text to a single line, but preserves paragraphs + """ + # remove initial line indent + text = textwrap.dedent(text) + # remove leading/trailing whitespace + text = text.strip() + + paragraphs = text.split("\n\n") + # remove consecutive whitespace + paragraphs = [re.sub(r"\s+", " ", paragraph) for paragraph in paragraphs] + return "\n\n".join(paragraphs) + + def parse_key_value_string( key_value_string: str, positional_arg_names: Sequence[str] | None = None, diff --git a/unit_test/utils_test.py b/unit_test/utils_test.py index 3c4a58859..8b0be33d3 100644 --- a/unit_test/utils_test.py +++ b/unit_test/utils_test.py @@ -12,6 +12,8 @@ format_safe, parse_key_value_string, prepare_command, + unwrap, + unwrap_preserving_paragraphs, ) from cibuildwheel.util.packaging import find_compatible_wheel @@ -363,3 +365,33 @@ def test_copy_test_sources_alternate_copy_into(sample_project): ], any_order=True, ) + + +def test_unwrap(): + assert ( + unwrap(""" + This is a + multiline + string + """) + == "This is a multiline string" + ) + + +def test_unwrap_preserving_paragraphs(): + assert ( + unwrap(""" + This is a + multiline + string + """) + == "This is a multiline string" + ) + assert ( + unwrap_preserving_paragraphs(""" + paragraph one + + paragraph two + """) + == "paragraph one\n\nparagraph two" + ) From 9d0f6bc63410aa5117aa7800af54995713bcdadb Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 21 Mar 2025 19:45:37 +0000 Subject: [PATCH 047/109] Add a schema entry --- bin/generate_schema.py | 3 ++ .../resources/cibuildwheel.schema.json | 36 +++++++++++++++---- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/bin/generate_schema.py b/bin/generate_schema.py index b82a91136..776869a09 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -181,6 +181,9 @@ musllinux-x86_64-image: type: string description: Specify alternative manylinux / musllinux container images + pyodide-version: + type: string + description: Specify the version of Pyodide to use repair-wheel-command: type: string_array description: Execute a shell command to repair each built wheel. diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 46af911c8..3da3af1f8 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -397,6 +397,11 @@ "description": "Specify alternative manylinux / musllinux container images", "title": "CIBW_MUSLLINUX_X86_64_IMAGE" }, + "pyodide-version": { + "type": "string", + "description": "Specify the version of Pyodide to use", + "title": "CIBW_PYODIDE_VERSION" + }, "repair-wheel-command": { "description": "Execute a shell command to repair each built wheel.", "oneOf": [ @@ -658,6 +663,9 @@ "musllinux-x86_64-image": { "$ref": "#/properties/musllinux-x86_64-image" }, + "pyodide-version": { + "$ref": "#/properties/pyodide-version" + }, "repair-wheel-command": { "$ref": "#/properties/repair-wheel-command" }, @@ -758,6 +766,9 @@ "musllinux-x86_64-image": { "$ref": "#/properties/musllinux-x86_64-image" }, + "pyodide-version": { + "$ref": "#/properties/pyodide-version" + }, "repair-wheel-command": { "description": "Execute a shell command to repair each built wheel.", "oneOf": [ @@ -822,6 +833,9 @@ "environment": { "$ref": "#/properties/environment" }, + "pyodide-version": { + "$ref": "#/properties/pyodide-version" + }, "repair-wheel-command": { "$ref": "#/properties/repair-wheel-command" }, @@ -873,6 +887,9 @@ "environment": { "$ref": "#/properties/environment" }, + "pyodide-version": { + "$ref": "#/properties/pyodide-version" + }, "repair-wheel-command": { "description": "Execute a shell command to repair each built wheel.", "oneOf": [ @@ -937,6 +954,9 @@ "environment": { "$ref": "#/properties/environment" }, + "pyodide-version": { + "$ref": "#/properties/pyodide-version" + }, "repair-wheel-command": { "$ref": "#/properties/repair-wheel-command" }, @@ -954,11 +974,6 @@ }, "test-requires": { "$ref": "#/properties/test-requires" - }, - "pyodide-version": { - "description": "Specify the Pyodide xbuildenv version to use for building", - "type": "string", - "title": "CIBW_PYODIDE_VERSION" } } }, @@ -993,6 +1008,9 @@ "environment": { "$ref": "#/properties/environment" }, + "pyodide-version": { + "$ref": "#/properties/pyodide-version" + }, "repair-wheel-command": { "$ref": "#/properties/repair-wheel-command" }, @@ -1002,10 +1020,16 @@ "test-extras": { "$ref": "#/properties/test-extras" }, + "test-sources": { + "$ref": "#/properties/test-sources" + }, + "test-groups": { + "$ref": "#/properties/test-groups" + }, "test-requires": { "$ref": "#/properties/test-requires" } - } + } } } } From b9f55e1ec5e601debe78776e1c90b89ac59143da Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 24 Mar 2025 21:50:18 +0000 Subject: [PATCH 048/109] Add docs for CIBW_PYODIDE_VERSION --- docs/options.md | 39 +++++++++++++++++++++++++++++++++++++++ docs/platforms/pyodide.md | 7 ++----- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/docs/options.md b/docs/options.md index 53a1ad27d..59010f31f 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1431,6 +1431,45 @@ Platform-specific environment variables are also available:
dependency-versions = { packages = ["pyodide-build==0.29.1"] } ``` +### `CIBW_PYODIDE_VERSION` {: #pyodide-version} + +> Specify the Pyodide version to use for `pyodide` platform builds + +This option allows you to specify a specific version of Pyodide to be used when building wheels for the `pyodide` platform. By default, cibuildwheel will use a default Pyodide version compatible with the current cibuildwheel and `pyodide-build` versions. + +This option is particularly useful for: + +- Testing against specific Pyodide alpha or older releases. +- Ensuring reproducibility by targeting a known Pyodide version. + +The available Pyodide versions are determined by the version of `pyodide-build` being used. You can list the compatible versions using the command `pyodide xbuildenv search --all` as described in the [Pyodide platform documentation](platforms/pyodide.md#choosing-a-version). + +!!! tip + You can set the version of `pyodide-build` using the [`CIBW_DEPENDENCY_VERSIONS`](#dependency-versions) option. + +#### Examples + +!!! tab examples "Environment variables" + + ```yaml + # Build Pyodide wheels using Pyodide version 0.26.4 + CIBW_PYODIDE_VERSION: 0.26.4 + + # Build Pyodide wheels using a specific alpha release + CIBW_PYODIDE_VERSION: 0.27.0a2 + ``` + +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel.pyodide] + # Build Pyodide wheels using Pyodide version 0.26.4 + pyodide-version = "0.26.4" + + [tool.cibuildwheel.pyodide] + # Build Pyodide wheels using a specific alpha release + pyodide-version = "0.27.0a2" + ``` ## Testing diff --git a/docs/platforms/pyodide.md b/docs/platforms/pyodide.md index 38a5afc70..672dbbd69 100644 --- a/docs/platforms/pyodide.md +++ b/docs/platforms/pyodide.md @@ -12,9 +12,6 @@ You need to have a matching host version of Python (unlike all other cibuildwhee You must target pyodide with `--platform pyodide` (or use `--only` on the identifier). -!!!tip +## Choosing a Pyodide version {: #choosing-a-version} - It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` environment variable to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json) or can be - referenced using the `pyodide xbuildenv search` command. - - This option is **not available** in the `pyproject.toml` config. +It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` option to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json), which can be viewed more easily by installing `pyodide-build` from PyPI and use `pyodide xbuildenv search --all` to see a compatibility table. From 611b032dcd7866de6ca9a3dbb017bfc4625e9eac Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 24 Mar 2025 21:54:35 +0000 Subject: [PATCH 049/109] Rephrase --- docs/options.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index 59010f31f..56c86c2dc 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1435,7 +1435,7 @@ Platform-specific environment variables are also available:
> Specify the Pyodide version to use for `pyodide` platform builds -This option allows you to specify a specific version of Pyodide to be used when building wheels for the `pyodide` platform. By default, cibuildwheel will use a default Pyodide version compatible with the current cibuildwheel and `pyodide-build` versions. +This option allows you to specify a specific version of Pyodide to be used when building wheels for the `pyodide` platform. If unset, cibuildwheel will use a pinned Pyodide version. This option is particularly useful for: From 16f4c5d947ca2a2f498d6969311ee072e862b989 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 24 Mar 2025 22:09:42 +0000 Subject: [PATCH 050/109] Add tests for pyodide-version --- test/test_emscripten.py | 28 ++++++++++++++++++++++++++++ unit_test/options_test.py | 5 +++++ 2 files changed, 33 insertions(+) diff --git a/test/test_emscripten.py b/test/test_emscripten.py index 897cd67f1..87a97516c 100644 --- a/test/test_emscripten.py +++ b/test/test_emscripten.py @@ -1,4 +1,5 @@ import shutil +import subprocess import sys import textwrap @@ -84,3 +85,30 @@ def test_pyodide_build(tmp_path, use_pyproject_toml): print("expected_wheels", expected_wheels) assert set(actual_wheels) == set(expected_wheels) + + +def test_pyodide_version_incompatible(tmp_path, capfd): + if sys.platform == "win32": + pytest.skip("emsdk doesn't work correctly on Windows") + + if not shutil.which("python3.12"): + pytest.skip("Python 3.12 not installed") + + if detect_ci_provider() == CIProvider.travis_ci: + pytest.skip("Python 3.12 is just a non-working pyenv shim") + + basic_project.generate(tmp_path) + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run( + tmp_path, + add_args=["--platform", "pyodide"], + add_env={ + "CIBW_DEPENDENCY_VERSIONS": "packages: pyodide-build==0.29.3", + "CIBW_PYODIDE_VERSION": "0.26.0a6", + }, + ) + + out, err = capfd.readouterr() + + assert "is not compatible with the pyodide-build version" in err diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 499862a99..55384f149 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -30,6 +30,8 @@ environment-pass = ["EXAMPLE_ENV"] +pyodide-version = "0.26.4" + [tool.cibuildwheel.macos] test-requires = "else" @@ -84,6 +86,9 @@ def test_options_1(tmp_path, monkeypatch): assert local.test_sources == ["test", "other dir"] assert local.manylinux_images["x86_64"] == pinned_x86_64_container_image["manylinux2014"] + local = options.build_options("cp312-pyodide_wasm32") + assert local.pyodide_version == "0.26.4" + def test_passthrough(tmp_path, monkeypatch): with tmp_path.joinpath("pyproject.toml").open("w") as f: From aef35c9a01e8bcf86fad14c9f7d55281a420e592 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 26 Mar 2025 13:56:50 +0000 Subject: [PATCH 051/109] Apply suggestions from code review --- docs/options.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/options.md b/docs/options.md index 32a225525..3a4670b83 100644 --- a/docs/options.md +++ b/docs/options.md @@ -256,7 +256,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. - For `ios` you need to be running on macOS, with Xcode and the iOS simulator installed. -- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may succeed, though there are known bugs. See [the section on Pyodide](setup.md#pyodide-(WebAssembly)-builds-(experimental)) for more information. +- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may succeed, though there are known bugs. See [the section on Pyodide](platforms/pyodide.md) for more information. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. @@ -308,7 +308,7 @@ The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425] Windows arm64 platform support is experimental. For an experimental WebAssembly build with `--platform pyodide`, -`cp312-pyodide_wasm32` is the only platform identifier, corresponding to [Pyodide version `0.26.4`](https://github.com/pyodide/pyodide/releases/tag/0.26.4). +`cp312-pyodide_wasm32` is the only platform identifier, corresponding to [Pyodide version `0.27.0`](https://github.com/pyodide/pyodide/releases/tag/0.27.0). See the [cibuildwheel 1 documentation](https://cibuildwheel.pypa.io/en/1.x/) for past end-of-life versions of Python, and PyPy2.7. From 10617753a50c1edc001ccb0b557005abbaf5c301 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 17:03:03 +0100 Subject: [PATCH 052/109] Add python_build_standalone util --- cibuildwheel/util/python_build_standalone.py | 204 +++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 cibuildwheel/util/python_build_standalone.py diff --git a/cibuildwheel/util/python_build_standalone.py b/cibuildwheel/util/python_build_standalone.py new file mode 100644 index 000000000..2d9169f76 --- /dev/null +++ b/cibuildwheel/util/python_build_standalone.py @@ -0,0 +1,204 @@ +import fnmatch +import json +import platform +import typing +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +from filelock import FileLock + +from cibuildwheel import __version__ as cibw_version +from cibuildwheel.util.file import download, extract_tar + + +class PythonBuildStandaloneError(Exception): + """Errors related to python-build-standalone.""" + + +def _get_platform_identifiers() -> tuple[str, str, str | None]: + """ + Detects the current platform and returns architecture, platform, and libc + identifiers. + """ + system = platform.system() + machine = platform.machine() + machine_lower = machine.lower() + + arch_identifier: str + platform_identifier: str + libc_identifier: str | None = None + + # Map Architecture + if machine_lower in ["x86_64", "amd64"]: + arch_identifier = "x86_64" + elif machine_lower in ["aarch64", "arm64"]: + arch_identifier = "aarch64" + else: + msg = f"Unsupported architecture: {system} {machine}. Cannot download appropriate Python build." + raise PythonBuildStandaloneError(msg) + + # Map OS + Libc + if system == "Linux": + platform_identifier = "unknown-linux" + libc_identifier = "musl" if "musl" in (platform.libc_ver() or ("", "")) else "gnu" + elif system == "Darwin": + platform_identifier = "apple-darwin" + elif system == "Windows": + platform_identifier = "pc-windows-msvc" + else: + msg = f"Unsupported operating system: {system}. Cannot download appropriate Python build." + raise PythonBuildStandaloneError(msg) + + print( + f"Detected platform: arch='{arch_identifier}', platform='{platform_identifier}', libc='{libc_identifier}'" + ) + return arch_identifier, platform_identifier, libc_identifier + + +def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: + """ + Makes a GitHub API request to the given path and returns the JSON response. + """ + api_url = f"https://api.github.com/{path}" + headers = { + "Accept": "application/vnd.github.v3+json", + "User-Agent": f"cibuildwheel/{cibw_version}", + } + request = urllib.request.Request(api_url, headers=headers) + + try: + with urllib.request.urlopen(request, timeout=30) as response: + return typing.cast(dict[str, Any], json.load(response)) + + except (urllib.error.URLError, TimeoutError) as e: + if max_retries > 0: + print(f"Retrying GitHub API request due to error: {e}") + return github_api_request(path, max_retries=max_retries - 1) + else: + msg = f"GitHub API request failed (Network error: {e}). Check network connection." + + raise PythonBuildStandaloneError(msg) from e + + +def _find_matching_asset_url( + *, + release_tag: str, + release_data: dict[str, Any], + python_version: str, + arch_identifier: str, + platform_identifier: str, + libc_identifier: str | None, +) -> tuple[str, str]: + """Finds the matching asset URL and filename.""" + assets = release_data.get("assets", []) + + if not assets: + msg = f"No download assets found in release {release_tag}." + raise PythonBuildStandaloneError(msg) + + expected_suffix = f"{arch_identifier}-{platform_identifier}" + if libc_identifier: + expected_suffix += f"-{libc_identifier}" + expected_suffix += "-install_only.tar.gz" + + asset_pattern = f"cpython-{python_version}.*-{expected_suffix}" + print(f"Looking for file with pattern {asset_pattern}'") + + for asset in assets: + asset_name = asset.get("name") + if not asset_name or not isinstance(asset_name, str): + continue + + if not fnmatch.fnmatch(asset_name, asset_pattern): + continue + + asset_url = asset.get("browser_download_url") + if asset_url and isinstance(asset_url, str): + return asset_url, asset_name + + # If loop completes without finding a match + msg = f"Could not find python-build-standalone release asset matching {asset_pattern!r}." + raise PythonBuildStandaloneError(msg) + + +def _download_or_get_from_cache(asset_url: str, asset_filename: str, cache_dir: Path) -> Path: + with FileLock(cache_dir / (asset_filename + ".lock")): + asset_cache_path = cache_dir / asset_filename + if asset_cache_path.is_file(): + print(f"Using cached python_build_standalone: {asset_cache_path}") + return asset_cache_path + + print(f"Downloading python_build_standalone: {asset_url} to {asset_cache_path}") + download(asset_url, asset_cache_path) + return asset_cache_path + + +def _find_python_executable(extracted_dir: Path) -> Path: + """Finds the python executable within the extracted directory structure. Raises FatalError.""" + # Structure is typically 'python/bin/python' or 'python/python.exe' + base_install_dir = extracted_dir / "python" + + if platform.system() == "Windows": + executable_path = base_install_dir / "python.exe" + else: + executable_path = base_install_dir / "bin" / "python" + + if not executable_path.is_file(): + msg = f"Could not locate python executable at expected path {executable_path} within {extracted_dir}." + raise PythonBuildStandaloneError(msg) + + print(f"Found python executable: {executable_path}") + return executable_path.resolve() # Return absolute path + + +def create_python_build_standalone_environment( + release_tag: str, python_version: str, temp_dir: Path, cache_dir: Path +) -> str: + """ + Returns a Python environment from python-build-standalone, + downloading it if necessary using a cache, and expanding it into a fresh base path. + + Args: + release_tag: The exact GitHub release tag (e.g., "20240224"). + python_version: The Python version string (e.g., "3.12"). + temp_dir: A directory where the Python environment will be extracted. + cache_dir: A directory to store/retrieve downloaded archives. + + Returns: + The absolute path to the python executable within the extracted environment (in temp_dir). + + Raises: + PythonBuildStandaloneError: If the platform is unsupported, the build cannot be found, + download/extraction fails, or configuration is invalid. + """ + + print( + f"Creating python-build-standalone environment: version={python_version}, tag={release_tag}" + ) + + arch_id, platform_id, libc_id = _get_platform_identifiers() + release_data = github_api_request( + f"repos/astral-sh/python-build-standalone/releases/tags/{release_tag}" + ) + + asset_url, asset_filename = _find_matching_asset_url( + release_tag=release_tag, + release_data=release_data, + python_version=python_version, + arch_identifier=arch_id, + platform_identifier=platform_id, + libc_identifier=libc_id, + ) + + archive_path = _download_or_get_from_cache( + asset_url=asset_url, asset_filename=asset_filename, cache_dir=cache_dir + ) + + python_base_dir = temp_dir / f"pbs-{release_tag}-{python_version}" + assert not python_base_dir.exists() + extract_tar(archive_path, python_base_dir) + + executable_path = _find_python_executable(python_base_dir) + return str(executable_path) From 5e3a8eda1bb35521e767fe2539cf1d8925e0d044 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 17:59:38 +0100 Subject: [PATCH 053/109] Hook up to python-build-standalone, removing dependency on host python --- cibuildwheel/platforms/pyodide.py | 35 +++++++++++--------- cibuildwheel/resources/build-platforms.toml | 2 +- cibuildwheel/util/python_build_standalone.py | 5 ++- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index e676b3821..68651a77c 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -32,6 +32,10 @@ ) from ..util.helpers import prepare_command, unwrap, unwrap_preserving_paragraphs from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version +from ..util.python_build_standalone import ( + PythonBuildStandaloneError, + create_python_build_standalone_environment, +) from ..venv import constraint_flags, virtualenv IS_WIN: Final[bool] = sys.platform.startswith("win") @@ -43,6 +47,7 @@ class PythonConfiguration: identifier: str default_pyodide_version: str node_version: str + python_build_standalone_tag: str class PyodideXBuildEnvInfoVersionRange(TypedDict): @@ -206,20 +211,20 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v return str(pyodide_root) -def get_base_python(identifier: str) -> Path: - implementation_id = identifier.split("-")[0] - majorminor = implementation_id[len("cp") :] - version_info = (int(majorminor[0]), int(majorminor[1:])) - if version_info == sys.version_info[:2]: - return Path(sys.executable) - - major_minor = ".".join(str(v) for v in version_info) - python_name = f"python{major_minor}" - which_python = shutil.which(python_name) - if which_python is None: - msg = f"CPython {major_minor} is not installed." - raise errors.FatalError(msg) - return Path(which_python) +def get_base_python(tmp: Path, python_configuration: PythonConfiguration) -> Path: + try: + return create_python_build_standalone_environment( + release_tag=python_configuration.python_build_standalone_tag, + python_version=python_configuration.version, + temp_dir=tmp, + cache_dir=CIBW_CACHE_PATH, + ) + except PythonBuildStandaloneError as e: + msg = unwrap(f""" + Failed to create a Python build environment: + {e} + """) + raise errors.FatalError(msg) from e def setup_python( @@ -229,7 +234,7 @@ def setup_python( environment: ParsedEnvironment, user_pyodide_version: str | None, ) -> dict[str, str]: - base_python = get_base_python(python_configuration.identifier) + base_python = get_base_python(tmp / "base", python_configuration) pyodide_version = user_pyodide_version or python_configuration.default_pyodide_version log.step("Setting up build environment...") diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 43caf085d..1e32f4a13 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -161,7 +161,7 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.0", node_version = "v22" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.0", node_version = "v22", python_build_standalone_tag = "20250317" }, ] [ios] diff --git a/cibuildwheel/util/python_build_standalone.py b/cibuildwheel/util/python_build_standalone.py index 2d9169f76..03bf274a4 100644 --- a/cibuildwheel/util/python_build_standalone.py +++ b/cibuildwheel/util/python_build_standalone.py @@ -155,7 +155,7 @@ def _find_python_executable(extracted_dir: Path) -> Path: def create_python_build_standalone_environment( release_tag: str, python_version: str, temp_dir: Path, cache_dir: Path -) -> str: +) -> Path: """ Returns a Python environment from python-build-standalone, downloading it if necessary using a cache, and expanding it into a fresh base path. @@ -200,5 +200,4 @@ def create_python_build_standalone_environment( assert not python_base_dir.exists() extract_tar(archive_path, python_base_dir) - executable_path = _find_python_executable(python_base_dir) - return str(executable_path) + return _find_python_executable(python_base_dir) From 985fc463b9c2f6d4e506b70fb78eea355a8e4455 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 9 Apr 2025 18:07:54 +0100 Subject: [PATCH 054/109] Remove python hard-code in action.yml --- action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/action.yml b/action.yml index 4f8ff6796..3e19ab982 100644 --- a/action.yml +++ b/action.yml @@ -24,11 +24,10 @@ branding: runs: using: composite steps: - # Set up the version of Python that supports pyodide - uses: actions/setup-python@v5 id: python with: - python-version: "3.12" + python-version: "3.11 - 3.13" update-environment: false - id: cibw From 62b57e66f973bf9e8410af9264fcf561e56908c5 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 11 Apr 2025 11:43:57 +0100 Subject: [PATCH 055/109] Add log step --- cibuildwheel/platforms/pyodide.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index 68651a77c..98a0cce8f 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -234,10 +234,11 @@ def setup_python( environment: ParsedEnvironment, user_pyodide_version: str | None, ) -> dict[str, str]: + log.step("Installing a base python environment...") base_python = get_base_python(tmp / "base", python_configuration) - pyodide_version = user_pyodide_version or python_configuration.default_pyodide_version log.step("Setting up build environment...") + pyodide_version = user_pyodide_version or python_configuration.default_pyodide_version venv_path = tmp / "venv" env = virtualenv(python_configuration.version, base_python, venv_path, None, use_uv=False) venv_bin_path = venv_path / "bin" From b3ee8d137f3bb0cb26c15d44818eddc9b51d2242 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 11 Apr 2025 11:44:25 +0100 Subject: [PATCH 056/109] Add workaround for https://github.com/pyodide/pyodide-build/issues/143 --- cibuildwheel/platforms/pyodide.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index 98a0cce8f..b69a10797 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -3,6 +3,7 @@ import os import shutil import sys +import textwrap import tomllib import typing from collections.abc import Set @@ -476,6 +477,32 @@ def build(options: Options, tmp_path: Path) -> None: call("pyodide", "venv", venv_dir, env=virtualenv_create_env) + # WORKAROUND (should be upstreamed into pyodide) + base_python = (identifier_tmp_dir / "build" / "venv" / "bin" / "python").resolve() + (venv_dir / "bin" / "python-host-link").symlink_to(base_python) + (venv_dir / "bin" / "python-host").unlink() + (venv_dir / "bin" / "python-host").write_text( + textwrap.dedent(f""" + #!/bin/bash + export PYTHONHOME={base_python.parent.parent} + exec {venv_dir / "bin" / "python-host-link"} -s "$@" + """) + ) + (venv_dir / "bin" / "python-host").chmod(0o755) + + # make the python3.12-host bin point to the python-host script + (venv_dir / "bin" / "python3.12-host").unlink() + (venv_dir / "bin" / "python3.12-host").symlink_to(venv_dir / "bin" / "python-host") + + # make the first line of the pip executable point to the + # python-host script the `/usr/bin/env` bit is needed to use a + # script as a shebang interpreter, at least on macOS + pip_executable = venv_dir / "bin" / "pip" + pip_executable_content = pip_executable.read_text().splitlines() + pip_executable_content[0] = f"#!/usr/bin/env {venv_dir / 'bin' / 'python-host'}" + pip_executable.write_text("\n".join(pip_executable_content)) + # END WORKAROUND + virtualenv_env["PATH"] = os.pathsep.join( [ str(venv_dir / "bin"), From 8b8cd6230884ae9ea97c5b3227ffe8b8555642c3 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 11 Apr 2025 13:01:28 +0100 Subject: [PATCH 057/109] Add emscripten pytest test --- test/test_emscripten.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/test/test_emscripten.py b/test/test_emscripten.py index 87a97516c..234bedb25 100644 --- a/test/test_emscripten.py +++ b/test/test_emscripten.py @@ -1,3 +1,4 @@ +import contextlib import shutil import subprocess import sys @@ -112,3 +113,43 @@ def test_pyodide_version_incompatible(tmp_path, capfd): out, err = capfd.readouterr() assert "is not compatible with the pyodide-build version" in err + + +@pytest.mark.parametrize("expect_failure", [True, False]) +def test_pyodide_build_and_test(tmp_path, expect_failure): + if sys.platform == "win32": + pytest.skip("emsdk doesn't work correctly on Windows") + + if expect_failure: + basic_project.files["test/spam_test.py"] = textwrap.dedent(r""" + def test_filter(): + assert 0 == 1 + """) + else: + basic_project.files["test/spam_test.py"] = textwrap.dedent(r""" + import spam + def test_filter(): + assert spam.filter("spam") == 0 + """) + basic_project.generate(tmp_path) + + context = ( + pytest.raises(subprocess.CalledProcessError) if expect_failure else contextlib.nullcontext() + ) + with context: + # build the wheels + actual_wheels = utils.cibuildwheel_run( + tmp_path, + add_args=["--platform", "pyodide"], + add_env={ + "CIBW_TEST_REQUIRES": "pytest", + "CIBW_TEST_COMMAND": "python -m pytest", + }, + ) + # check that the expected wheels are produced + expected_wheels = [ + "spam-0.1.0-cp312-cp312-pyodide_2024_0_wasm32.whl", + ] + print("actual_wheels", actual_wheels) + print("expected_wheels", expected_wheels) + assert set(actual_wheels) == set(expected_wheels) From 50773978e890a7108baad5fee182341be3f22519 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 11 Apr 2025 13:02:03 +0100 Subject: [PATCH 058/109] Remove unneeded checks --- test/test_emscripten.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/test/test_emscripten.py b/test/test_emscripten.py index 234bedb25..48aaff956 100644 --- a/test/test_emscripten.py +++ b/test/test_emscripten.py @@ -1,12 +1,10 @@ import contextlib -import shutil import subprocess import sys import textwrap import pytest -from cibuildwheel.ci import CIProvider, detect_ci_provider from cibuildwheel.util.file import CIBW_CACHE_PATH from . import test_projects, utils @@ -47,12 +45,6 @@ def test_pyodide_build(tmp_path, use_pyproject_toml): if sys.platform == "win32": pytest.skip("emsdk doesn't work correctly on Windows") - if not shutil.which("python3.12"): - pytest.skip("Python 3.12 not installed") - - if detect_ci_provider() == CIProvider.travis_ci: - pytest.skip("Python 3.12 is just a non-working pyenv shim") - if use_pyproject_toml: basic_project.files["pyproject.toml"] = textwrap.dedent( """ @@ -92,12 +84,6 @@ def test_pyodide_version_incompatible(tmp_path, capfd): if sys.platform == "win32": pytest.skip("emsdk doesn't work correctly on Windows") - if not shutil.which("python3.12"): - pytest.skip("Python 3.12 not installed") - - if detect_ci_provider() == CIProvider.travis_ci: - pytest.skip("Python 3.12 is just a non-working pyenv shim") - basic_project.generate(tmp_path) with pytest.raises(subprocess.CalledProcessError): From 3efb38fe34cc77b9a456cf7e5cb62db11425da59 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 11 Apr 2025 13:02:23 +0100 Subject: [PATCH 059/109] Fix a pytest invoke for emscripten --- test/test_testing.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/test_testing.py b/test/test_testing.py index a4bae870c..5bf03d376 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -210,8 +210,7 @@ def test_test_sources(tmp_path): project_dir, add_env={ "CIBW_TEST_REQUIRES": "pytest", - "CIBW_TEST_COMMAND": "pytest", - "CIBW_TEST_COMMAND_WINDOWS": "pytest", + "CIBW_TEST_COMMAND": utils.invoke_pytest(), "CIBW_TEST_SOURCES": "test", }, ) From 468dd9226dfbab0c68047284bba5f81efcc751f0 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 20 Apr 2025 18:35:20 +0100 Subject: [PATCH 060/109] Generate pyodide-build constraints from the pinned pyodide version --- .../resources/constraints-pyodide312.txt | 24 +++++----- .../resources/constraints-python310.txt | 4 +- .../resources/constraints-python311.txt | 4 +- .../resources/constraints-python312.txt | 4 +- .../resources/constraints-python313.txt | 4 +- .../resources/constraints-python38.txt | 4 +- .../resources/constraints-python39.txt | 4 +- cibuildwheel/resources/constraints.txt | 4 +- noxfile.py | 47 +++++++++++++++++-- 9 files changed, 70 insertions(+), 29 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index aed02adf7..9eba4e43a 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -4,7 +4,7 @@ annotated-types==0.7.0 # via pydantic anyio==4.9.0 # via httpx -auditwheel-emscripten==0.0.16 +auditwheel-emscripten==0.1.0 # via pyodide-build build==1.2.2.post1 # via @@ -19,15 +19,13 @@ charset-normalizer==3.4.1 # via requests click==8.1.8 # via typer -cmake==4.0.0 - # via pyodide-build distlib==0.3.9 # via virtualenv filelock==3.18.0 # via virtualenv h11==0.14.0 # via httpcore -httpcore==1.0.7 +httpcore==1.0.8 # via httpx httpx==0.28.1 # via unearth @@ -42,7 +40,7 @@ markdown-it-py==3.0.0 # via rich mdurl==0.1.2 # via markdown-it-py -packaging==24.2 +packaging==25.0 # via # auditwheel-emscripten # build @@ -51,8 +49,10 @@ packaging==24.2 pip==25.0.1 # via -r .nox/update_constraints/tmp/constraints-pyodide.in platformdirs==4.3.7 - # via virtualenv -pydantic==2.11.2 + # via + # pyodide-build + # virtualenv +pydantic==2.11.3 # via # pyodide-build # pyodide-lock @@ -60,9 +60,9 @@ pydantic-core==2.33.1 # via pydantic pygments==2.19.1 # via rich -pyodide-build==0.29.2 +pyodide-build @ git+https://github.com/pyodide/pyodide-build.git@71a6d09651814439feb161537888078f7ff68c19 # via -r .nox/update_constraints/tmp/constraints-pyodide.in -pyodide-cli==0.2.4 +pyodide-cli==0.3.0 # via # auditwheel-emscripten # pyodide-build @@ -92,7 +92,7 @@ typer==0.15.2 # auditwheel-emscripten # pyodide-build # pyodide-cli -typing-extensions==4.13.1 +typing-extensions==4.13.2 # via # anyio # pydantic @@ -101,9 +101,9 @@ typing-extensions==4.13.1 # typing-inspection typing-inspection==0.4.0 # via pydantic -unearth==0.17.3 +unearth==0.17.5 # via pyodide-build -urllib3==2.3.0 +urllib3==2.4.0 # via requests virtualenv==20.30.0 # via diff --git a/cibuildwheel/resources/constraints-python310.txt b/cibuildwheel/resources/constraints-python310.txt index 49a2b619f..9eccc27be 100644 --- a/cibuildwheel/resources/constraints-python310.txt +++ b/cibuildwheel/resources/constraints-python310.txt @@ -14,7 +14,7 @@ importlib-metadata==8.6.1 # via build macholib==1.16.3 # via delocate -packaging==24.2 +packaging==25.0 # via # build # delocate @@ -26,7 +26,7 @@ pyproject-hooks==1.2.0 # via build tomli==2.2.1 # via build -typing-extensions==4.13.1 +typing-extensions==4.13.2 # via delocate virtualenv==20.30.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python311.txt b/cibuildwheel/resources/constraints-python311.txt index 524d961e2..7abb00bfa 100644 --- a/cibuildwheel/resources/constraints-python311.txt +++ b/cibuildwheel/resources/constraints-python311.txt @@ -12,7 +12,7 @@ filelock==3.18.0 # via virtualenv macholib==1.16.3 # via delocate -packaging==24.2 +packaging==25.0 # via # build # delocate @@ -22,7 +22,7 @@ platformdirs==4.3.7 # via virtualenv pyproject-hooks==1.2.0 # via build -typing-extensions==4.13.1 +typing-extensions==4.13.2 # via delocate virtualenv==20.30.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python312.txt b/cibuildwheel/resources/constraints-python312.txt index 524d961e2..7abb00bfa 100644 --- a/cibuildwheel/resources/constraints-python312.txt +++ b/cibuildwheel/resources/constraints-python312.txt @@ -12,7 +12,7 @@ filelock==3.18.0 # via virtualenv macholib==1.16.3 # via delocate -packaging==24.2 +packaging==25.0 # via # build # delocate @@ -22,7 +22,7 @@ platformdirs==4.3.7 # via virtualenv pyproject-hooks==1.2.0 # via build -typing-extensions==4.13.1 +typing-extensions==4.13.2 # via delocate virtualenv==20.30.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python313.txt b/cibuildwheel/resources/constraints-python313.txt index 524d961e2..7abb00bfa 100644 --- a/cibuildwheel/resources/constraints-python313.txt +++ b/cibuildwheel/resources/constraints-python313.txt @@ -12,7 +12,7 @@ filelock==3.18.0 # via virtualenv macholib==1.16.3 # via delocate -packaging==24.2 +packaging==25.0 # via # build # delocate @@ -22,7 +22,7 @@ platformdirs==4.3.7 # via virtualenv pyproject-hooks==1.2.0 # via build -typing-extensions==4.13.1 +typing-extensions==4.13.2 # via delocate virtualenv==20.30.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python38.txt b/cibuildwheel/resources/constraints-python38.txt index 9ea26b911..490e373a9 100644 --- a/cibuildwheel/resources/constraints-python38.txt +++ b/cibuildwheel/resources/constraints-python38.txt @@ -14,7 +14,7 @@ importlib-metadata==8.5.0 # via build macholib==1.16.3 # via delocate -packaging==24.2 +packaging==25.0 # via # build # delocate @@ -26,7 +26,7 @@ pyproject-hooks==1.2.0 # via build tomli==2.2.1 # via build -typing-extensions==4.13.1 +typing-extensions==4.13.2 # via delocate virtualenv==20.30.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python39.txt b/cibuildwheel/resources/constraints-python39.txt index 49a2b619f..9eccc27be 100644 --- a/cibuildwheel/resources/constraints-python39.txt +++ b/cibuildwheel/resources/constraints-python39.txt @@ -14,7 +14,7 @@ importlib-metadata==8.6.1 # via build macholib==1.16.3 # via delocate -packaging==24.2 +packaging==25.0 # via # build # delocate @@ -26,7 +26,7 @@ pyproject-hooks==1.2.0 # via build tomli==2.2.1 # via build -typing-extensions==4.13.1 +typing-extensions==4.13.2 # via delocate virtualenv==20.30.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index 524d961e2..7abb00bfa 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -12,7 +12,7 @@ filelock==3.18.0 # via virtualenv macholib==1.16.3 # via delocate -packaging==24.2 +packaging==25.0 # via # build # delocate @@ -22,7 +22,7 @@ platformdirs==4.3.7 # via virtualenv pyproject-hooks==1.2.0 # via build -typing-extensions==4.13.1 +typing-extensions==4.13.2 # via delocate virtualenv==20.30.0 # via -r cibuildwheel/resources/constraints.in diff --git a/noxfile.py b/noxfile.py index 356bb8946..fe7cc673d 100755 --- a/noxfile.py +++ b/noxfile.py @@ -15,9 +15,12 @@ See sessions with `nox -l` """ +import json import os import shutil import sys +import textwrap +import urllib.request from pathlib import Path import nox @@ -100,10 +103,48 @@ def update_constraints(session: nox.Session) -> None: pyodides = build_platforms["pyodide"]["python_configurations"] for pyodide in pyodides: python_version = ".".join(pyodide["version"].split(".")[:2]) - pyodide_build_version = pyodide["pyodide_build_version"] - output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt" + pyodide_version = pyodide["default_pyodide_version"] + + xbuildenv_info_url = "https://raw.githubusercontent.com/pyodide/pyodide/refs/heads/main/pyodide-cross-build-environments.json" + with urllib.request.urlopen(xbuildenv_info_url) as response: + xbuildenv_info = json.loads(response.read().decode("utf-8")) + + pyodide_version_xbuildenv_info = xbuildenv_info["releases"][pyodide_version] + + pyodide_build_min_version = pyodide_version_xbuildenv_info.get("min_pyodide_build_version") + pyodide_build_max_version = pyodide_version_xbuildenv_info.get("max_pyodide_build_version") + + pyodide_build_specifier_parts: list[str] = [] + + if pyodide_build_min_version: + pyodide_build_specifier_parts.append(f">={pyodide_build_min_version}") + if pyodide_build_max_version: + pyodide_build_specifier_parts.append(f"<={pyodide_build_max_version}") + + pyodide_build_specifier = ",".join(pyodide_build_specifier_parts) + tmp_file = Path(session.create_tmp()) / "constraints-pyodide.in" - tmp_file.write_text(f"pip\nbuild[virtualenv]\npyodide-build=={pyodide_build_version}") + # tmp_file.write_text( + # textwrap.dedent(f""" + # pip + # build[virtualenv] + # pyodide-build{pyodide_build_specifier} + # """) + # ) + + # TODO: remove the pyodide-build git version pinning when the next release is + # available + print(f"pyodide-build specifier: {pyodide_build_specifier}") + tmp_file.write_text( + textwrap.dedent(""" + pip + build[virtualenv] + git+https://github.com/pyodide/pyodide-build.git@71a6d09651814439feb161537888078f7ff68c19 + """) + ) + # END TODO + + output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt" session.run( "uv", "pip", From da3ffad6ba62c8dc7ae7e0087b4b7ec677818cfe Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 20 Apr 2025 18:36:19 +0100 Subject: [PATCH 061/109] Remove pyodide python-build-standalone workaround --- cibuildwheel/platforms/pyodide.py | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index b69a10797..98a0cce8f 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -3,7 +3,6 @@ import os import shutil import sys -import textwrap import tomllib import typing from collections.abc import Set @@ -477,32 +476,6 @@ def build(options: Options, tmp_path: Path) -> None: call("pyodide", "venv", venv_dir, env=virtualenv_create_env) - # WORKAROUND (should be upstreamed into pyodide) - base_python = (identifier_tmp_dir / "build" / "venv" / "bin" / "python").resolve() - (venv_dir / "bin" / "python-host-link").symlink_to(base_python) - (venv_dir / "bin" / "python-host").unlink() - (venv_dir / "bin" / "python-host").write_text( - textwrap.dedent(f""" - #!/bin/bash - export PYTHONHOME={base_python.parent.parent} - exec {venv_dir / "bin" / "python-host-link"} -s "$@" - """) - ) - (venv_dir / "bin" / "python-host").chmod(0o755) - - # make the python3.12-host bin point to the python-host script - (venv_dir / "bin" / "python3.12-host").unlink() - (venv_dir / "bin" / "python3.12-host").symlink_to(venv_dir / "bin" / "python-host") - - # make the first line of the pip executable point to the - # python-host script the `/usr/bin/env` bit is needed to use a - # script as a shebang interpreter, at least on macOS - pip_executable = venv_dir / "bin" / "pip" - pip_executable_content = pip_executable.read_text().splitlines() - pip_executable_content[0] = f"#!/usr/bin/env {venv_dir / 'bin' / 'python-host'}" - pip_executable.write_text("\n".join(pip_executable_content)) - # END WORKAROUND - virtualenv_env["PATH"] = os.pathsep.join( [ str(venv_dir / "bin"), From 15b894a642e1ac75aee5bd9660e363804ab27339 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 20 Apr 2025 18:36:52 +0100 Subject: [PATCH 062/109] Fixup paths from newer version of pyodide-build --- cibuildwheel/platforms/pyodide.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index 98a0cce8f..082f4ed63 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -185,10 +185,8 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall # specify the pyodide-build version in the root path, which will set up the xbuildenv for # the requested Pyodide version. - pyodide_root = ( - CIBW_CACHE_PATH - / f".pyodide-xbuildenv-{pyodide_build_version}/{pyodide_version}/xbuildenv/pyodide-root" - ) + xbuildenv_cache_path = CIBW_CACHE_PATH / f"pyodide-xbuildenv-{pyodide_build_version}" + pyodide_root = xbuildenv_cache_path / pyodide_version / "xbuildenv" / "pyodide-root" with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): if pyodide_root.exists(): @@ -204,10 +202,14 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v "pyodide", "xbuildenv", "install", + "--path", + str(xbuildenv_cache_path), pyodide_version, env=env, cwd=CIBW_CACHE_PATH, ) + assert pyodide_root.exists() + return str(pyodide_root) From 5b860afa082e2fef34a390c568a9c4749cb28670 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 20 Apr 2025 18:56:53 +0100 Subject: [PATCH 063/109] Fix/skip some failing tests --- test/test_environment.py | 7 ++++++- test/test_pep518.py | 13 ++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/test/test_environment.py b/test/test_environment.py index 6e7ed6fd8..67b04829a 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -100,7 +100,12 @@ def test_overridden_path(tmp_path, capfd): "build_frontend", [ pytest.param("pip", marks=utils.skip_if_pyodide("No pip for pyodide")), - "build", + pytest.param( + "build", + marks=utils.skip_if_pyodide( + "pyodide doesn't support multiple values for PIP_CONSTRAINT" + ), + ), ], ) def test_overridden_pip_constraint(tmp_path, build_frontend): diff --git a/test/test_pep518.py b/test/test_pep518.py index 2147520f9..240c4949d 100644 --- a/test/test_pep518.py +++ b/test/test_pep518.py @@ -45,12 +45,19 @@ def test_pep518(tmp_path, build_frontend_env): assert not (project_dir / "42").exists() assert not (project_dir / "4.1.2").exists() - # pypa/build creates a "build" folder & a "*.egg-info" folder for the wheel being built, - # this should be harmless so remove them + # pypa/build creates a "build" folder & a "*.egg-info" folder for the + # wheel being built, this should be harmless so remove them. Pyodide build + # creates a ".pyodide_build" folder, but this is gitignored with a + # .gitignore file inside. contents = [ item for item in project_dir.iterdir() - if item.name != "build" and not item.name.endswith(".egg-info") + if item.name != "build" + and not item.name.endswith(".egg-info") + and item.name != ".pyodide_build" ] + print("Project contents after build:") + print("\n".join(f" {f}" for f in contents)) + assert len(contents) == len(basic_project.files) From ecc37a12eb76bb9d657e16d95e3a44de6cdb2f9e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 21 Apr 2025 10:52:42 +0100 Subject: [PATCH 064/109] Use `python -m pytest` on pyodide, even on Linux --- test/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/utils.py b/test/utils.py index b3d2b172d..11f7ac8b6 100644 --- a/test/utils.py +++ b/test/utils.py @@ -363,7 +363,7 @@ def skip_if_pyodide(reason: str) -> Any: def invoke_pytest() -> str: # see https://github.com/pyodide/pyodide/issues/4802 - if platform == "pyodide" and sys.platform.startswith("darwin"): + if platform == "pyodide": return "python -m pytest" return "pytest" From fd805ae97fe20a7d28a88909322cd5371ae99ec1 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 21 Apr 2025 22:58:01 +0100 Subject: [PATCH 065/109] Docs fixes --- docs/options.md | 4 ++-- docs/platforms.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/options.md b/docs/options.md index 69a33847d..742e211d9 100644 --- a/docs/options.md +++ b/docs/options.md @@ -17,7 +17,7 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. - For `ios` you need to be running on macOS, with Xcode and the iOS simulator installed. -- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may succeed, though there are known bugs. See [the section on Pyodide](platforms/pyodide.md) for more information. +- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may succeed, though there are known bugs. See [the section on Pyodide](platforms.md#pyodide) for more information. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. @@ -1223,7 +1223,7 @@ This option is particularly useful for: - Testing against specific Pyodide alpha or older releases. - Ensuring reproducibility by targeting a known Pyodide version. -The available Pyodide versions are determined by the version of `pyodide-build` being used. You can list the compatible versions using the command `pyodide xbuildenv search --all` as described in the [Pyodide platform documentation](platforms/pyodide.md#choosing-a-version). +The available Pyodide versions are determined by the version of `pyodide-build` being used. You can list the compatible versions using the command `pyodide xbuildenv search --all` as described in the [Pyodide platform documentation](platforms.md#pyodide-choosing-a-version). !!! tip You can set the version of `pyodide-build` using the [`CIBW_DEPENDENCY_VERSIONS`](#dependency-versions) option. diff --git a/docs/platforms.md b/docs/platforms.md index ba6364a85..44178f950 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -160,7 +160,7 @@ By default, `ARM64` is not enabled when running on non-ARM64 runners. Use [`CIBW Pyodide is offered as an experimental feature in cibuildwheel. -### Prerequisites +### System requirements You need to have a matching host version of Python (unlike all other cibuildwheel platforms). Linux host highly recommended; macOS hosts may work (e.g. invoking `pytest` directly in [`CIBW_TEST_COMMAND`](options.md#test-command) is [currently failing](https://github.com/pyodide/pyodide/issues/4802)) and Windows hosts will not work. @@ -168,7 +168,7 @@ You need to have a matching host version of Python (unlike all other cibuildwhee You must target pyodide with `--platform pyodide` (or use `--only` on the identifier). -### Choosing a Pyodide version {: #choosing-a-version} +### Choosing a Pyodide version {: #pyodide-choosing-a-version} It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` option to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json), which can be viewed more easily by installing `pyodide-build` from PyPI and use `pyodide xbuildenv search --all` to see a compatibility table. From c8fbd0b1817b9d283a270111da7feb61ab8f6f98 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 26 Apr 2025 13:10:19 +0100 Subject: [PATCH 066/109] Don't call the github API at runtime, cache the release assets instead --- bin/update_python_build_standalone.py | 59 +++ cibuildwheel/extra.py | 43 +- cibuildwheel/platforms/pyodide.py | 2 - cibuildwheel/resources/build-platforms.toml | 2 +- .../python-build-standalone-releases.json | 425 ++++++++++++++++++ cibuildwheel/util/python_build_standalone.py | 110 ++--- cibuildwheel/util/resources.py | 1 + noxfile.py | 4 +- 8 files changed, 575 insertions(+), 71 deletions(-) create mode 100755 bin/update_python_build_standalone.py create mode 100644 cibuildwheel/resources/python-build-standalone-releases.json diff --git a/bin/update_python_build_standalone.py b/bin/update_python_build_standalone.py new file mode 100755 index 000000000..516920144 --- /dev/null +++ b/bin/update_python_build_standalone.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import json + +from cibuildwheel.extra import github_api_request +from cibuildwheel.util.python_build_standalone import ( + PythonBuildStandaloneAsset, + PythonBuildStandaloneReleaseData, +) +from cibuildwheel.util.resources import PYTHON_BUILD_STANDALONE_RELEASES + + +def main() -> None: + """ + This script updates the vendored list of release assets to the latest + version of astral-sh/python-build-standalone. + """ + + # Get the latest release tag from the GitHub API + latest_release = github_api_request("repos/astral-sh/python-build-standalone/releases/latest") + latest_tag = latest_release["tag_name"] + + # Get the list of assets for the latest release + github_assets = github_api_request( + f"repos/astral-sh/python-build-standalone/releases/tags/{latest_tag}" + )["assets"] + + assets: list[PythonBuildStandaloneAsset] = [] + + for github_asset in github_assets: + name = github_asset["name"] + if not name.endswith("install_only.tar.gz"): + continue + url = github_asset["browser_download_url"] + assets.append({"name": name, "url": url}) + + # Write the assets to the JSON file. One day, we might need to support + # multiple releases, but for now, we only support the latest one + json_file_contents: PythonBuildStandaloneReleaseData = { + "releases": [ + { + "tag": latest_tag, + "assets": assets, + } + ] + } + + with PYTHON_BUILD_STANDALONE_RELEASES.open("w", encoding="utf-8") as f: + json.dump(json_file_contents, f, indent=2) + # Add a trailing newline, our pre-commit hook requires it + f.write("\n") + + print( + f"Updated {PYTHON_BUILD_STANDALONE_RELEASES.name} with {len(assets)} assets for tag {latest_tag}" + ) + + +if __name__ == "__main__": + main() diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py index 774447c33..f3755e36f 100644 --- a/cibuildwheel/extra.py +++ b/cibuildwheel/extra.py @@ -2,9 +2,16 @@ These are utilities for the `/bin` scripts, not for the `cibuildwheel` program. """ +import json +import time +import typing +import urllib.error +import urllib.request from collections.abc import Mapping, Sequence from io import StringIO -from typing import Protocol +from typing import Any, Protocol + +from cibuildwheel import __version__ as cibw_version __all__ = ("Printable", "dump_python_configurations") @@ -30,3 +37,37 @@ def dump_python_configurations( output.write("\n") # Strip the final newline, to avoid two blank lines at the end. return output.getvalue()[:-1] + + +def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: + """ + Makes a GitHub API request to the given path and returns the JSON response. + """ + api_url = f"https://api.github.com/{path}" + headers = { + "Accept": "application/vnd.github.v3+json", + "User-Agent": f"cibuildwheel/{cibw_version}", + } + request = urllib.request.Request(api_url, headers=headers) + + try: + with urllib.request.urlopen(request, timeout=30) as response: + return typing.cast(dict[str, Any], json.load(response)) + + except (urllib.error.URLError, TimeoutError) as e: + if max_retries > 0: + if ( + isinstance(e, urllib.error.HTTPError) + and (e.code == 403 or e.code == 429) + and e.get("x-ratelimit-remaining") == "0" + ): + reset_time = int(e.get("x-ratelimit-reset", 0)) + wait_time = max(0, reset_time - int(e.get("date", 0))) + print(f"Github rate limit exceeded. Waiting for {wait_time} seconds.") + time.sleep(wait_time) + else: + print(f"Retrying GitHub API request due to error: {e}") + return github_api_request(path, max_retries=max_retries - 1) + else: + print(f"GitHub API request failed (Network error: {e}). Check network connection.") + raise e diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index 082f4ed63..c049793c1 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -47,7 +47,6 @@ class PythonConfiguration: identifier: str default_pyodide_version: str node_version: str - python_build_standalone_tag: str class PyodideXBuildEnvInfoVersionRange(TypedDict): @@ -216,7 +215,6 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v def get_base_python(tmp: Path, python_configuration: PythonConfiguration) -> Path: try: return create_python_build_standalone_environment( - release_tag=python_configuration.python_build_standalone_tag, python_version=python_configuration.version, temp_dir=tmp, cache_dir=CIBW_CACHE_PATH, diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 0171e8e32..e48878a06 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -161,7 +161,7 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.0", node_version = "v22", python_build_standalone_tag = "20250317" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.0", node_version = "v22" }, ] [ios] diff --git a/cibuildwheel/resources/python-build-standalone-releases.json b/cibuildwheel/resources/python-build-standalone-releases.json new file mode 100644 index 000000000..407b0318c --- /dev/null +++ b/cibuildwheel/resources/python-build-standalone-releases.json @@ -0,0 +1,425 @@ +{ + "releases": [ + { + "tag": "20250409", + "assets": [ + { + "name": "cpython-3.10.17+20250409-aarch64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-aarch64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-aarch64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-aarch64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-armv7-unknown-linux-gnueabi-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-armv7-unknown-linux-gnueabi-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-armv7-unknown-linux-gnueabihf-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-armv7-unknown-linux-gnueabihf-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-i686-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-i686-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-ppc64le-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-riscv64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-riscv64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-s390x-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-s390x-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-x86_64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-x86_64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-x86_64-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-x86_64-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-x86_64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-x86_64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-x86_64-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-x86_64-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-x86_64_v2-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-x86_64_v2-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-x86_64_v2-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-x86_64_v2-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-x86_64_v3-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-x86_64_v3-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-x86_64_v3-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-x86_64_v4-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-x86_64_v4-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.10.17+20250409-x86_64_v4-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.10.17%2B20250409-x86_64_v4-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-aarch64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-aarch64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-aarch64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-aarch64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-armv7-unknown-linux-gnueabi-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-armv7-unknown-linux-gnueabi-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-armv7-unknown-linux-gnueabihf-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-armv7-unknown-linux-gnueabihf-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-i686-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-i686-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-ppc64le-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-riscv64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-riscv64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-s390x-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-s390x-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-x86_64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-x86_64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-x86_64-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-x86_64-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-x86_64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-x86_64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-x86_64-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-x86_64-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-x86_64_v2-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-x86_64_v2-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-x86_64_v2-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-x86_64_v2-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-x86_64_v3-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-x86_64_v3-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-x86_64_v3-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-x86_64_v4-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-x86_64_v4-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.11.12+20250409-x86_64_v4-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.11.12%2B20250409-x86_64_v4-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-aarch64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-aarch64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-aarch64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-aarch64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-armv7-unknown-linux-gnueabi-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-armv7-unknown-linux-gnueabi-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-armv7-unknown-linux-gnueabihf-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-armv7-unknown-linux-gnueabihf-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-i686-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-i686-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-ppc64le-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-riscv64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-riscv64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-s390x-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-s390x-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-x86_64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-x86_64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-x86_64-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-x86_64-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-x86_64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-x86_64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-x86_64-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-x86_64-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-x86_64_v2-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-x86_64_v2-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-x86_64_v2-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-x86_64_v2-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-x86_64_v3-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-x86_64_v3-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-x86_64_v3-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-x86_64_v4-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-x86_64_v4-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.12.10+20250409-x86_64_v4-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.12.10%2B20250409-x86_64_v4-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-aarch64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-aarch64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-aarch64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-aarch64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-armv7-unknown-linux-gnueabi-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-armv7-unknown-linux-gnueabi-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-armv7-unknown-linux-gnueabihf-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-armv7-unknown-linux-gnueabihf-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-i686-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-i686-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-ppc64le-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-riscv64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-riscv64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-s390x-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-s390x-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-x86_64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-x86_64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-x86_64-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-x86_64-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-x86_64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-x86_64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-x86_64-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-x86_64-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-x86_64_v2-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-x86_64_v2-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-x86_64_v2-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-x86_64_v2-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-x86_64_v3-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-x86_64_v3-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-x86_64_v3-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-x86_64_v4-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-x86_64_v4-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.13.3+20250409-x86_64_v4-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.13.3%2B20250409-x86_64_v4-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-aarch64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-aarch64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-aarch64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-aarch64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-armv7-unknown-linux-gnueabi-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-armv7-unknown-linux-gnueabi-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-armv7-unknown-linux-gnueabihf-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-armv7-unknown-linux-gnueabihf-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-i686-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-i686-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-ppc64le-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-riscv64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-riscv64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-s390x-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-s390x-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-x86_64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-x86_64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-x86_64-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-x86_64-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-x86_64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-x86_64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-x86_64_v2-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-x86_64_v2-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-x86_64_v3-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.14.0a6+20250409-x86_64_v4-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.14.0a6%2B20250409-x86_64_v4-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-aarch64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-aarch64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-aarch64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-aarch64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-armv7-unknown-linux-gnueabi-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-armv7-unknown-linux-gnueabi-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-armv7-unknown-linux-gnueabihf-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-armv7-unknown-linux-gnueabihf-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-i686-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-i686-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-ppc64le-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-ppc64le-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-riscv64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-riscv64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-s390x-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-s390x-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-x86_64-apple-darwin-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-x86_64-apple-darwin-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-x86_64-pc-windows-msvc-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-x86_64-pc-windows-msvc-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-x86_64-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-x86_64-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-x86_64-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-x86_64-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-x86_64_v2-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-x86_64_v2-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-x86_64_v2-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-x86_64_v2-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-x86_64_v3-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-x86_64_v3-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-x86_64_v3-unknown-linux-musl-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-x86_64_v4-unknown-linux-gnu-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-x86_64_v4-unknown-linux-gnu-install_only.tar.gz" + }, + { + "name": "cpython-3.9.22+20250409-x86_64_v4-unknown-linux-musl-install_only.tar.gz", + "url": "https://github.com/astral-sh/python-build-standalone/releases/download/20250409/cpython-3.9.22%2B20250409-x86_64_v4-unknown-linux-musl-install_only.tar.gz" + } + ] + } + ] +} diff --git a/cibuildwheel/util/python_build_standalone.py b/cibuildwheel/util/python_build_standalone.py index 03bf274a4..3e053bfcd 100644 --- a/cibuildwheel/util/python_build_standalone.py +++ b/cibuildwheel/util/python_build_standalone.py @@ -1,16 +1,34 @@ import fnmatch +import functools import json import platform import typing -import urllib.error -import urllib.request from pathlib import Path -from typing import Any from filelock import FileLock -from cibuildwheel import __version__ as cibw_version from cibuildwheel.util.file import download, extract_tar +from cibuildwheel.util.resources import PYTHON_BUILD_STANDALONE_RELEASES + + +class PythonBuildStandaloneAsset(typing.TypedDict): + name: str + url: str + + +class PythonBuildStandaloneRelease(typing.TypedDict): + tag: str + assets: list[PythonBuildStandaloneAsset] + + +class PythonBuildStandaloneReleaseData(typing.TypedDict): + releases: list[PythonBuildStandaloneRelease] + + +@functools.cache +def get_python_build_standalone_release_data() -> PythonBuildStandaloneReleaseData: + with open(PYTHON_BUILD_STANDALONE_RELEASES, "rb") as f: + return typing.cast(PythonBuildStandaloneReleaseData, json.load(f)) class PythonBuildStandaloneError(Exception): @@ -57,46 +75,15 @@ def _get_platform_identifiers() -> tuple[str, str, str | None]: return arch_identifier, platform_identifier, libc_identifier -def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: - """ - Makes a GitHub API request to the given path and returns the JSON response. - """ - api_url = f"https://api.github.com/{path}" - headers = { - "Accept": "application/vnd.github.v3+json", - "User-Agent": f"cibuildwheel/{cibw_version}", - } - request = urllib.request.Request(api_url, headers=headers) - - try: - with urllib.request.urlopen(request, timeout=30) as response: - return typing.cast(dict[str, Any], json.load(response)) - - except (urllib.error.URLError, TimeoutError) as e: - if max_retries > 0: - print(f"Retrying GitHub API request due to error: {e}") - return github_api_request(path, max_retries=max_retries - 1) - else: - msg = f"GitHub API request failed (Network error: {e}). Check network connection." - - raise PythonBuildStandaloneError(msg) from e - - -def _find_matching_asset_url( +def _get_pbs_asset( *, - release_tag: str, - release_data: dict[str, Any], python_version: str, arch_identifier: str, platform_identifier: str, libc_identifier: str | None, -) -> tuple[str, str]: - """Finds the matching asset URL and filename.""" - assets = release_data.get("assets", []) - - if not assets: - msg = f"No download assets found in release {release_tag}." - raise PythonBuildStandaloneError(msg) +) -> tuple[str, str, str]: + """Finds the asset, returning (tag, filename, url).""" + release_data = get_python_build_standalone_release_data() expected_suffix = f"{arch_identifier}-{platform_identifier}" if libc_identifier: @@ -106,17 +93,14 @@ def _find_matching_asset_url( asset_pattern = f"cpython-{python_version}.*-{expected_suffix}" print(f"Looking for file with pattern {asset_pattern}'") - for asset in assets: - asset_name = asset.get("name") - if not asset_name or not isinstance(asset_name, str): - continue + for release in release_data["releases"]: + for asset in release["assets"]: + asset_name = asset["name"] + if not fnmatch.fnmatch(asset_name, asset_pattern): + continue - if not fnmatch.fnmatch(asset_name, asset_pattern): - continue - - asset_url = asset.get("browser_download_url") - if asset_url and isinstance(asset_url, str): - return asset_url, asset_name + asset_url = asset["url"] + return release["tag"], asset_url, asset_name # If loop completes without finding a match msg = f"Could not find python-build-standalone release asset matching {asset_pattern!r}." @@ -136,7 +120,7 @@ def _download_or_get_from_cache(asset_url: str, asset_filename: str, cache_dir: def _find_python_executable(extracted_dir: Path) -> Path: - """Finds the python executable within the extracted directory structure. Raises FatalError.""" + """Finds the python executable within the extracted directory structure.""" # Structure is typically 'python/bin/python' or 'python/python.exe' base_install_dir = extracted_dir / "python" @@ -154,49 +138,43 @@ def _find_python_executable(extracted_dir: Path) -> Path: def create_python_build_standalone_environment( - release_tag: str, python_version: str, temp_dir: Path, cache_dir: Path + python_version: str, temp_dir: Path, cache_dir: Path ) -> Path: """ - Returns a Python environment from python-build-standalone, - downloading it if necessary using a cache, and expanding it into a fresh base path. + Returns a Python environment from python-build-standalone, downloading it + if necessary using a cache, and expanding it into a fresh base path. Args: - release_tag: The exact GitHub release tag (e.g., "20240224"). python_version: The Python version string (e.g., "3.12"). - temp_dir: A directory where the Python environment will be extracted. + temp_dir: A directory where the Python environment will be created. cache_dir: A directory to store/retrieve downloaded archives. Returns: - The absolute path to the python executable within the extracted environment (in temp_dir). + The absolute path to the python executable within the created environment (in temp_dir). Raises: PythonBuildStandaloneError: If the platform is unsupported, the build cannot be found, download/extraction fails, or configuration is invalid. """ - print( - f"Creating python-build-standalone environment: version={python_version}, tag={release_tag}" - ) + print(f"Creating python-build-standalone environment: version={python_version}") arch_id, platform_id, libc_id = _get_platform_identifiers() - release_data = github_api_request( - f"repos/astral-sh/python-build-standalone/releases/tags/{release_tag}" - ) - asset_url, asset_filename = _find_matching_asset_url( - release_tag=release_tag, - release_data=release_data, + pbs_tag, asset_url, asset_filename = _get_pbs_asset( python_version=python_version, arch_identifier=arch_id, platform_identifier=platform_id, libc_identifier=libc_id, ) + print(f"Using python-build-standalone release: {pbs_tag}") + archive_path = _download_or_get_from_cache( asset_url=asset_url, asset_filename=asset_filename, cache_dir=cache_dir ) - python_base_dir = temp_dir / f"pbs-{release_tag}-{python_version}" + python_base_dir = temp_dir / "pbs" assert not python_base_dir.exists() extract_tar(archive_path, python_base_dir) diff --git a/cibuildwheel/util/resources.py b/cibuildwheel/util/resources.py index e27f1f892..1080f1358 100644 --- a/cibuildwheel/util/resources.py +++ b/cibuildwheel/util/resources.py @@ -15,6 +15,7 @@ CONSTRAINTS: Final[Path] = PATH / "constraints.txt" VIRTUALENV: Final[Path] = PATH / "virtualenv.toml" CIBUILDWHEEL_SCHEMA: Final[Path] = PATH / "cibuildwheel.schema.json" +PYTHON_BUILD_STANDALONE_RELEASES: Final[Path] = PATH / "python-build-standalone-releases.json" # this value is cached because it's used a lot in unit tests diff --git a/noxfile.py b/noxfile.py index fe7cc673d..b6ea40a01 100755 --- a/noxfile.py +++ b/noxfile.py @@ -160,7 +160,8 @@ def update_constraints(session: nox.Session) -> None: @nox.session(default=False, tags=["update"]) def update_pins(session: nox.Session) -> None: """ - Update the python, docker and virtualenv pins version inplace. + Update the python, docker, virtualenv, node, and python-build-standalone + version pins inplace. """ pyproject = nox.project.load_toml() session.install("-e.", *nox.project.dependency_groups(pyproject, "bin")) @@ -168,6 +169,7 @@ def update_pins(session: nox.Session) -> None: session.run("python", "bin/update_docker.py") session.run("python", "bin/update_virtualenv.py", "--force") session.run("python", "bin/update_nodejs.py", "--force") + session.run("python", "bin/update_python_build_standalone.py") @nox.session(default=False, reuse_venv=True, tags=["update"]) From 89bdaa526efd5874ac4c9ef11bbfef54803def64 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 26 Apr 2025 13:16:42 +0100 Subject: [PATCH 067/109] Ignore pylint false positive --- cibuildwheel/extra.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py index f3755e36f..940d93bd7 100644 --- a/cibuildwheel/extra.py +++ b/cibuildwheel/extra.py @@ -55,6 +55,7 @@ def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: return typing.cast(dict[str, Any], json.load(response)) except (urllib.error.URLError, TimeoutError) as e: + # pylint: disable=E1101 if max_retries > 0: if ( isinstance(e, urllib.error.HTTPError) From 6f387f6a87358c8fb4a6167671867bc463d0ccb2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 26 Apr 2025 13:54:29 +0100 Subject: [PATCH 068/109] Add version auto-updating for pyodide --- bin/update_pythons.py | 53 ++++++++++++++++++++- cibuildwheel/extra.py | 20 +++++++- cibuildwheel/resources/build-platforms.toml | 2 +- cibuildwheel/resources/nodejs.toml | 4 +- 4 files changed, 73 insertions(+), 6 deletions(-) diff --git a/bin/update_pythons.py b/bin/update_pythons.py index 7b8ea1474..2dc3f2dac 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -18,7 +18,7 @@ from rich.logging import RichHandler from rich.syntax import Syntax -from cibuildwheel.extra import dump_python_configurations +from cibuildwheel.extra import dump_python_configurations, get_pyodide_xbuildenv_info log = logging.getLogger("cibw") @@ -50,7 +50,14 @@ class ConfigApple(TypedDict): url: str -AnyConfig = ConfigWinCP | ConfigWinPP | ConfigApple +class ConfigPyodide(TypedDict): + identifier: str + version: str + default_pyodide_version: str + node_version: str + + +AnyConfig = ConfigWinCP | ConfigWinPP | ConfigApple | ConfigPyodide # The following set of "Versions" classes allow the initial call to the APIs to @@ -274,6 +281,39 @@ def update_version_ios(self, identifier: str, version: Version) -> ConfigApple | return None +class PyodideVersions: + def __init__(self) -> None: + xbuildenv_info = get_pyodide_xbuildenv_info() + self.releases = xbuildenv_info["releases"] + + def update_version_pyodide( + self, identifier: str, version: Version, spec: Specifier, node_version: str + ) -> ConfigPyodide | None: + # get releases that match the python version + releases = [ + r for r in self.releases.values() if spec.contains(Version(r["python_version"])) + ] + # sort by version, latest first + releases.sort(key=lambda r: Version(r["version"]), reverse=True) + + if not releases: + msg = f"Pyodide not found for {spec}!" + raise ValueError(msg) + + final_releases = [r for r in releases if not Version(r["version"]).is_prerelease] + + # prefer a final release if available, otherwise use the latest + # pre-release + release = final_releases[0] if final_releases else releases[0] + + return ConfigPyodide( + identifier=identifier, + version=str(version), + default_pyodide_version=release["version"], + node_version=node_version, + ) + + # This is a universal interface to all the above Versions classes. Given an # identifier, it updates a config dict. @@ -294,6 +334,8 @@ def __init__(self) -> None: self.ios_cpython = CPythonIOSVersions() + self.pyodide = PyodideVersions() + def update_config(self, config: MutableMapping[str, str]) -> None: identifier = config["identifier"] version = Version(config["version"]) @@ -328,6 +370,10 @@ def update_config(self, config: MutableMapping[str, str]) -> None: config_update = self.windows_arm64.update_version_windows(spec) elif "ios" in identifier: config_update = self.ios_cpython.update_version_ios(identifier, version) + elif "pyodide" in identifier: + config_update = self.pyodide.update_version_pyodide( + identifier, version, spec, config["node_version"] + ) assert config_update is not None, f"{identifier} not found!" config.update(**config_update) @@ -366,6 +412,9 @@ def update_pythons(force: bool, level: str) -> None: for config in configs["ios"]["python_configurations"]: all_versions.update_config(config) + for config in configs["pyodide"]["python_configurations"]: + all_versions.update_config(config) + result_toml = dump_python_configurations(configs) rich.print() # spacer diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py index 940d93bd7..b8c7c42fb 100644 --- a/cibuildwheel/extra.py +++ b/cibuildwheel/extra.py @@ -9,7 +9,7 @@ import urllib.request from collections.abc import Mapping, Sequence from io import StringIO -from typing import Any, Protocol +from typing import Any, NotRequired, Protocol from cibuildwheel import __version__ as cibw_version @@ -72,3 +72,21 @@ def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: else: print(f"GitHub API request failed (Network error: {e}). Check network connection.") raise e + + +class PyodideXBuildEnvRelease(typing.TypedDict): + version: str + python_version: str + emscripten_version: str + min_pyodide_build_version: NotRequired[str] + max_pyodide_build_version: NotRequired[str] + + +class PyodideXBuildEnvInfo(typing.TypedDict): + releases: dict[str, PyodideXBuildEnvRelease] + + +def get_pyodide_xbuildenv_info() -> PyodideXBuildEnvInfo: + xbuildenv_info_url = "https://raw.githubusercontent.com/pyodide/pyodide/refs/heads/main/pyodide-cross-build-environments.json" + with urllib.request.urlopen(xbuildenv_info_url) as response: + return typing.cast(PyodideXBuildEnvInfo, json.loads(response.read().decode("utf-8"))) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index e48878a06..c89dea353 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -161,7 +161,7 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.0", node_version = "v22" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.5", node_version = "v22" }, ] [ios] diff --git a/cibuildwheel/resources/nodejs.toml b/cibuildwheel/resources/nodejs.toml index ad93f2d7b..9674bc39f 100644 --- a/cibuildwheel/resources/nodejs.toml +++ b/cibuildwheel/resources/nodejs.toml @@ -1,3 +1,3 @@ url = "https://nodejs.org/dist/" -v22 = "v22.14.0" -v20 = "v20.19.0" +v22 = "v22.15.0" +v20 = "v20.19.1" From 95dea1d2b56ef8a52dec8856573b4d71b496d76f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 26 Apr 2025 14:12:58 +0100 Subject: [PATCH 069/109] Add support for pyodide 3.13. --- cibuildwheel/resources/build-platforms.toml | 1 + docs/options.md | 27 +++++++++------------ test/test_emscripten.py | 2 ++ test/utils.py | 16 ++++++------ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index c89dea353..d8270d1cc 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -162,6 +162,7 @@ python_configurations = [ [pyodide] python_configurations = [ { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.5", node_version = "v22" }, + { identifier = "cp313-pyodide_wasm32", version = "3.13", default_pyodide_version = "0.28.0a1", node_version = "v22" }, ] [ios] diff --git a/docs/options.md b/docs/options.md index 742e211d9..2e7d96c52 100644 --- a/docs/options.md +++ b/docs/options.md @@ -50,27 +50,24 @@ When setting the options, you can use shell-style globbing syntax, as per [fnmat
-| | macOS | Windows | Linux Intel | Linux Other | iOS | -|---------------|------------------------------------------------------------------------|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------| -| Python 3.8 | cp38-macosx_x86_64
cp38-macosx_universal2
cp38-macosx_arm64 | cp38-win_amd64
cp38-win32 | cp38-manylinux_x86_64
cp38-manylinux_i686
cp38-musllinux_x86_64
cp38-musllinux_i686 | cp38-manylinux_aarch64
cp38-manylinux_ppc64le
cp38-manylinux_s390x
cp38-manylinux_armv7l
cp38-musllinux_aarch64
cp38-musllinux_ppc64le
cp38-musllinux_s390x
cp38-musllinux_armv7l | | -| Python 3.9 | cp39-macosx_x86_64
cp39-macosx_universal2
cp39-macosx_arm64 | cp39-win_amd64
cp39-win32
cp39-win_arm64 | cp39-manylinux_x86_64
cp39-manylinux_i686
cp39-musllinux_x86_64
cp39-musllinux_i686 | cp39-manylinux_aarch64
cp39-manylinux_ppc64le
cp39-manylinux_s390x
cp39-manylinux_armv7l
cp39-musllinux_aarch64
cp39-musllinux_ppc64le
cp39-musllinux_s390x
cp39-musllinux_armv7l | | -| Python 3.10 | cp310-macosx_x86_64
cp310-macosx_universal2
cp310-macosx_arm64 | cp310-win_amd64
cp310-win32
cp310-win_arm64 | cp310-manylinux_x86_64
cp310-manylinux_i686
cp310-musllinux_x86_64
cp310-musllinux_i686 | cp310-manylinux_aarch64
cp310-manylinux_ppc64le
cp310-manylinux_s390x
cp310-manylinux_armv7l
cp310-musllinux_aarch64
cp310-musllinux_ppc64le
cp310-musllinux_s390x
cp310-musllinux_armv7l | | -| Python 3.11 | cp311-macosx_x86_64
cp311-macosx_universal2
cp311-macosx_arm64 | cp311-win_amd64
cp311-win32
cp311-win_arm64 | cp311-manylinux_x86_64
cp311-manylinux_i686
cp311-musllinux_x86_64
cp311-musllinux_i686 | cp311-manylinux_aarch64
cp311-manylinux_ppc64le
cp311-manylinux_s390x
cp311-manylinux_armv7l
cp311-musllinux_aarch64
cp311-musllinux_ppc64le
cp311-musllinux_s390x
cp311-musllinux_armv7l | | -| Python 3.12 | cp312-macosx_x86_64
cp312-macosx_universal2
cp312-macosx_arm64 | cp312-win_amd64
cp312-win32
cp312-win_arm64 | cp312-manylinux_x86_64
cp312-manylinux_i686
cp312-musllinux_x86_64
cp312-musllinux_i686 | cp312-manylinux_aarch64
cp312-manylinux_ppc64le
cp312-manylinux_s390x
cp312-musllinux_armv7l
cp312-musllinux_ppc64le
cp312-musllinux_s390x
cp312-musllinux_armv7l | | -| Python 3.13 | cp313-macosx_x86_64
cp313-macosx_universal2
cp313-macosx_arm64 | cp313-win_amd64
cp313-win32
cp313-win_arm64 | cp313-manylinux_x86_64
cp313-manylinux_i686
cp313-musllinux_x86_64
cp313-musllinux_i686 | cp313-manylinux_aarch64
cp313-manylinux_ppc64le
cp313-manylinux_s390x
cp313-manylinux_armv7l
cp313-musllinux_aarch64
cp313-musllinux_ppc64le
cp313-musllinux_s390x
cp313-musllinux_armv7l | cp313-ios_arm64_iphoneos
cp313-ios_arm64_iphonesimulator
cp313-ios_x86_64_iphonesimulator | -| PyPy3.8 v7.3 | pp38-macosx_x86_64
pp38-macosx_arm64 | pp38-win_amd64 | pp38-manylinux_x86_64
pp38-manylinux_i686 | pp38-manylinux_aarch64 | | -| PyPy3.9 v7.3 | pp39-macosx_x86_64
pp39-macosx_arm64 | pp39-win_amd64 | pp39-manylinux_x86_64
pp39-manylinux_i686 | pp39-manylinux_aarch64 | | -| PyPy3.10 v7.3 | pp310-macosx_x86_64
pp310-macosx_arm64 | pp310-win_amd64 | pp310-manylinux_x86_64
pp310-manylinux_i686 | pp310-manylinux_aarch64 | | -| PyPy3.11 v7.3 | pp311-macosx_x86_64
pp311-macosx_arm64 | pp311-win_amd64 | pp311-manylinux_x86_64
pp311-manylinux_i686 | pp311-manylinux_aarch64 | | +| | macOS | Windows | Linux Intel | Linux Other | iOS | pyodide (WASM) | +|---------------|------------------------------------------------------------------------|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------|----------------------| +| Python 3.8 | cp38-macosx_x86_64
cp38-macosx_universal2
cp38-macosx_arm64 | cp38-win_amd64
cp38-win32 | cp38-manylinux_x86_64
cp38-manylinux_i686
cp38-musllinux_x86_64
cp38-musllinux_i686 | cp38-manylinux_aarch64
cp38-manylinux_ppc64le
cp38-manylinux_s390x
cp38-manylinux_armv7l
cp38-musllinux_aarch64
cp38-musllinux_ppc64le
cp38-musllinux_s390x
cp38-musllinux_armv7l | | | +| Python 3.9 | cp39-macosx_x86_64
cp39-macosx_universal2
cp39-macosx_arm64 | cp39-win_amd64
cp39-win32
cp39-win_arm64 | cp39-manylinux_x86_64
cp39-manylinux_i686
cp39-musllinux_x86_64
cp39-musllinux_i686 | cp39-manylinux_aarch64
cp39-manylinux_ppc64le
cp39-manylinux_s390x
cp39-manylinux_armv7l
cp39-musllinux_aarch64
cp39-musllinux_ppc64le
cp39-musllinux_s390x
cp39-musllinux_armv7l | | | +| Python 3.10 | cp310-macosx_x86_64
cp310-macosx_universal2
cp310-macosx_arm64 | cp310-win_amd64
cp310-win32
cp310-win_arm64 | cp310-manylinux_x86_64
cp310-manylinux_i686
cp310-musllinux_x86_64
cp310-musllinux_i686 | cp310-manylinux_aarch64
cp310-manylinux_ppc64le
cp310-manylinux_s390x
cp310-manylinux_armv7l
cp310-musllinux_aarch64
cp310-musllinux_ppc64le
cp310-musllinux_s390x
cp310-musllinux_armv7l | | | +| Python 3.11 | cp311-macosx_x86_64
cp311-macosx_universal2
cp311-macosx_arm64 | cp311-win_amd64
cp311-win32
cp311-win_arm64 | cp311-manylinux_x86_64
cp311-manylinux_i686
cp311-musllinux_x86_64
cp311-musllinux_i686 | cp311-manylinux_aarch64
cp311-manylinux_ppc64le
cp311-manylinux_s390x
cp311-manylinux_armv7l
cp311-musllinux_aarch64
cp311-musllinux_ppc64le
cp311-musllinux_s390x
cp311-musllinux_armv7l | | | +| Python 3.12 | cp312-macosx_x86_64
cp312-macosx_universal2
cp312-macosx_arm64 | cp312-win_amd64
cp312-win32
cp312-win_arm64 | cp312-manylinux_x86_64
cp312-manylinux_i686
cp312-musllinux_x86_64
cp312-musllinux_i686 | cp312-manylinux_aarch64
cp312-manylinux_ppc64le
cp312-manylinux_s390x
cp312-musllinux_armv7l
cp312-musllinux_ppc64le
cp312-musllinux_s390x
cp312-musllinux_armv7l | | cp312-pyodide_wasm32 | +| Python 3.13 | cp313-macosx_x86_64
cp313-macosx_universal2
cp313-macosx_arm64 | cp313-win_amd64
cp313-win32
cp313-win_arm64 | cp313-manylinux_x86_64
cp313-manylinux_i686
cp313-musllinux_x86_64
cp313-musllinux_i686 | cp313-manylinux_aarch64
cp313-manylinux_ppc64le
cp313-manylinux_s390x
cp313-manylinux_armv7l
cp313-musllinux_aarch64
cp313-musllinux_ppc64le
cp313-musllinux_s390x
cp313-musllinux_armv7l | cp313-ios_arm64_iphoneos
cp313-ios_arm64_iphonesimulator
cp313-ios_x86_64_iphonesimulator | cp313-pyodide_wasm32 | +| PyPy3.8 v7.3 | pp38-macosx_x86_64
pp38-macosx_arm64 | pp38-win_amd64 | pp38-manylinux_x86_64
pp38-manylinux_i686 | pp38-manylinux_aarch64 | | | +| PyPy3.9 v7.3 | pp39-macosx_x86_64
pp39-macosx_arm64 | pp39-win_amd64 | pp39-manylinux_x86_64
pp39-manylinux_i686 | pp39-manylinux_aarch64 | | | +| PyPy3.10 v7.3 | pp310-macosx_x86_64
pp310-macosx_arm64 | pp310-win_amd64 | pp310-manylinux_x86_64
pp310-manylinux_i686 | pp310-manylinux_aarch64 | | | +| PyPy3.11 v7.3 | pp311-macosx_x86_64
pp311-macosx_arm64 | pp311-win_amd64 | pp311-manylinux_x86_64
pp311-manylinux_i686 | pp311-manylinux_aarch64 | | | The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to cibuildwheel. The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details). Windows arm64 platform support is experimental. -For an experimental WebAssembly build with `--platform pyodide`, -`cp312-pyodide_wasm32` is the only platform identifier, corresponding to [Pyodide version `0.27.0`](https://github.com/pyodide/pyodide/releases/tag/0.27.0). - See the [cibuildwheel 2 documentation](https://cibuildwheel.pypa.io/en/2.x/) for past end-of-life versions of Python. #### Examples diff --git a/test/test_emscripten.py b/test/test_emscripten.py index 48aaff956..b122883d7 100644 --- a/test/test_emscripten.py +++ b/test/test_emscripten.py @@ -72,6 +72,7 @@ def test_pyodide_build(tmp_path, use_pyproject_toml): # check that the expected wheels are produced expected_wheels = [ "spam-0.1.0-cp312-cp312-pyodide_2024_0_wasm32.whl", + "spam-0.1.0-cp313-cp313-pyodide_2025_0_wasm32.whl", ] print("actual_wheels", actual_wheels) @@ -135,6 +136,7 @@ def test_filter(): # check that the expected wheels are produced expected_wheels = [ "spam-0.1.0-cp312-cp312-pyodide_2024_0_wasm32.whl", + "spam-0.1.0-cp313-cp313-pyodide_2025_0_wasm32.whl", ] print("actual_wheels", actual_wheels) print("expected_wheels", expected_wheels) diff --git a/test/utils.py b/test/utils.py index 11f7ac8b6..3c6c57c20 100644 --- a/test/utils.py +++ b/test/utils.py @@ -233,7 +233,7 @@ def _expected_wheels( musllinux_versions = ["musllinux_1_2"] if platform == "pyodide" and python_abi_tags is None: - python_abi_tags = ["cp312-cp312"] + python_abi_tags = ["cp312-cp312", "cp313-cp313"] if python_abi_tags is None: python_abi_tags = [ "cp38-cp38", @@ -267,13 +267,6 @@ def _expected_wheels( ) ] - if platform == "pyodide": - assert len(python_abi_tags) == 1 - python_abi_tag = python_abi_tags[0] - platform_tag = "pyodide_2024_0_wasm32" - yield f"{package_name}-{package_version}-{python_abi_tag}-{platform_tag}.whl" - return - for python_abi_tag in python_abi_tags: platform_tags = [] @@ -322,6 +315,13 @@ def _expected_wheels( if include_universal2: platform_tags.append(f"macosx_{min_macosx.replace('.', '_')}_universal2") + + elif platform == "pyodide": + platform_tags = { + "cp312-cp312": ["pyodide_2024_0_wasm32"], + "cp313-cp313": ["pyodide_2025_0_wasm32"], + }.get(python_abi_tag, []) + else: msg = f"Unsupported platform {platform!r}" raise Exception(msg) From c9ee5a09f4c204be715e88ffeb06a910f3349278 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 26 Apr 2025 14:21:32 +0100 Subject: [PATCH 070/109] Fix tests for multiple pyodide wheels --- cibuildwheel/util/packaging.py | 3 +++ test/test_abi_variants.py | 3 ++- test/test_custom_repair_wheel.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/util/packaging.py b/cibuildwheel/util/packaging.py index f6619dd92..310778198 100644 --- a/cibuildwheel/util/packaging.py +++ b/cibuildwheel/util/packaging.py @@ -165,6 +165,9 @@ def find_compatible_wheel(wheels: Sequence[T], identifier: str) -> T | None: continue if not tag.platform.endswith(f"_{arch}"): continue + elif platform.startswith("pyodide"): + # each Pyodide version has its own platform tag + continue else: # Windows should exactly match if tag.platform != platform: diff --git a/test/test_abi_variants.py b/test/test_abi_variants.py index ec529849d..af4918110 100644 --- a/test/test_abi_variants.py +++ b/test/test_abi_variants.py @@ -54,6 +54,7 @@ def test_abi3(tmp_path): expected_wheels = [ w.replace(f"{single_python_tag}-{single_python_tag}", "cp310-abi3") for w in expected_wheels + if single_python_tag in w ] else: expected_wheels = [ @@ -194,7 +195,7 @@ def test_abi_none(tmp_path, capfd): captured = capfd.readouterr() assert "Building wheel..." in captured.out if utils.platform == "pyodide": - # there's only 1 possible configuration for pyodide, we won't see the message expected on following builds + # pyodide has a unique platform tag for each wheel, so they are not reused assert "Found previously built wheel" not in captured.out else: assert "Found previously built wheel" in captured.out diff --git a/test/test_custom_repair_wheel.py b/test/test_custom_repair_wheel.py index bd22d6343..cd04bc688 100644 --- a/test/test_custom_repair_wheel.py +++ b/test/test_custom_repair_wheel.py @@ -48,7 +48,7 @@ def test(tmp_path, capfd): assert "Build failed because a wheel named" in captured.err assert exc_info.value.returncode == 6 else: - # We only produced one wheel (currently Pyodide) + # We only produced one wheel (perhaps Pyodide) # check that it has the right name # # As far as I can tell, this is the only full test coverage for From e440e111a1e427961c92811a4e086f7f3bfa8e32 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 26 Apr 2025 14:24:14 +0100 Subject: [PATCH 071/109] Remove workaround for unreleased pyodide-build --- .../resources/constraints-pyodide312.txt | 10 +- .../resources/constraints-pyodide313.txt | 112 ++++++++++++++++++ noxfile.py | 16 +-- 3 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 cibuildwheel/resources/constraints-pyodide313.txt diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 9eba4e43a..f1a346156 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -10,7 +10,7 @@ build==1.2.2.post1 # via # -r .nox/update_constraints/tmp/constraints-pyodide.in # pyodide-build -certifi==2025.1.31 +certifi==2025.4.26 # via # httpcore # httpx @@ -23,9 +23,9 @@ distlib==0.3.9 # via virtualenv filelock==3.18.0 # via virtualenv -h11==0.14.0 +h11==0.16.0 # via httpcore -httpcore==1.0.8 +httpcore==1.0.9 # via httpx httpx==0.28.1 # via unearth @@ -46,7 +46,7 @@ packaging==25.0 # build # pyodide-build # unearth -pip==25.0.1 +pip==25.1 # via -r .nox/update_constraints/tmp/constraints-pyodide.in platformdirs==4.3.7 # via @@ -60,7 +60,7 @@ pydantic-core==2.33.1 # via pydantic pygments==2.19.1 # via rich -pyodide-build @ git+https://github.com/pyodide/pyodide-build.git@71a6d09651814439feb161537888078f7ff68c19 +pyodide-build==0.30.1 # via -r .nox/update_constraints/tmp/constraints-pyodide.in pyodide-cli==0.3.0 # via diff --git a/cibuildwheel/resources/constraints-pyodide313.txt b/cibuildwheel/resources/constraints-pyodide313.txt new file mode 100644 index 000000000..995b0046f --- /dev/null +++ b/cibuildwheel/resources/constraints-pyodide313.txt @@ -0,0 +1,112 @@ +# This file was autogenerated by uv via the following command: +# nox -s update_constraints +annotated-types==0.7.0 + # via pydantic +anyio==4.9.0 + # via httpx +auditwheel-emscripten==0.1.0 + # via pyodide-build +build==1.2.2.post1 + # via + # -r .nox/update_constraints/tmp/constraints-pyodide.in + # pyodide-build +certifi==2025.4.26 + # via + # httpcore + # httpx + # requests +charset-normalizer==3.4.1 + # via requests +click==8.1.8 + # via typer +distlib==0.3.9 + # via virtualenv +filelock==3.18.0 + # via virtualenv +h11==0.16.0 + # via httpcore +httpcore==1.0.9 + # via httpx +httpx==0.28.1 + # via unearth +idna==3.10 + # via + # anyio + # httpx + # requests +leb128==1.0.8 + # via auditwheel-emscripten +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +packaging==25.0 + # via + # auditwheel-emscripten + # build + # pyodide-build + # unearth +pip==25.1 + # via -r .nox/update_constraints/tmp/constraints-pyodide.in +platformdirs==4.3.7 + # via + # pyodide-build + # virtualenv +pydantic==2.11.3 + # via + # pyodide-build + # pyodide-lock +pydantic-core==2.33.1 + # via pydantic +pygments==2.19.1 + # via rich +pyodide-build==0.30.1 + # via -r .nox/update_constraints/tmp/constraints-pyodide.in +pyodide-cli==0.3.0 + # via + # auditwheel-emscripten + # pyodide-build +pyodide-lock==0.1.0a7 + # via pyodide-build +pyproject-hooks==1.2.0 + # via build +requests==2.32.3 + # via pyodide-build +resolvelib==1.1.0 + # via pyodide-build +rich==14.0.0 + # via + # pyodide-build + # pyodide-cli + # typer +ruamel-yaml==0.18.10 + # via pyodide-build +shellingham==1.5.4 + # via typer +sniffio==1.3.1 + # via anyio +typer==0.15.2 + # via + # auditwheel-emscripten + # pyodide-build + # pyodide-cli +typing-extensions==4.13.2 + # via + # pydantic + # pydantic-core + # typer + # typing-inspection +typing-inspection==0.4.0 + # via pydantic +unearth==0.17.5 + # via pyodide-build +urllib3==2.4.0 + # via requests +virtualenv==20.30.0 + # via + # build + # pyodide-build +wheel==0.45.1 + # via + # auditwheel-emscripten + # pyodide-build diff --git a/noxfile.py b/noxfile.py index b6ea40a01..6faf7ed62 100755 --- a/noxfile.py +++ b/noxfile.py @@ -124,25 +124,13 @@ def update_constraints(session: nox.Session) -> None: pyodide_build_specifier = ",".join(pyodide_build_specifier_parts) tmp_file = Path(session.create_tmp()) / "constraints-pyodide.in" - # tmp_file.write_text( - # textwrap.dedent(f""" - # pip - # build[virtualenv] - # pyodide-build{pyodide_build_specifier} - # """) - # ) - - # TODO: remove the pyodide-build git version pinning when the next release is - # available - print(f"pyodide-build specifier: {pyodide_build_specifier}") tmp_file.write_text( - textwrap.dedent(""" + textwrap.dedent(f""" pip build[virtualenv] - git+https://github.com/pyodide/pyodide-build.git@71a6d09651814439feb161537888078f7ff68c19 + pyodide-build{pyodide_build_specifier} """) ) - # END TODO output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt" session.run( From 4f78dd8b4106628dadf3ecde5ea84197f1d17780 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 26 Apr 2025 14:25:11 +0100 Subject: [PATCH 072/109] Rename to "test_pyodide" --- test/{test_emscripten.py => test_pyodide.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{test_emscripten.py => test_pyodide.py} (100%) diff --git a/test/test_emscripten.py b/test/test_pyodide.py similarity index 100% rename from test/test_emscripten.py rename to test/test_pyodide.py From 6f3e6ca709d1cf6c2d02a482e06142afc955d52a Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 27 Apr 2025 20:02:31 +0100 Subject: [PATCH 073/109] Fix pathname confusion --- cibuildwheel/platforms/pyodide.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index c049793c1..c4a05e9cd 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -184,7 +184,7 @@ def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_v # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall # specify the pyodide-build version in the root path, which will set up the xbuildenv for # the requested Pyodide version. - xbuildenv_cache_path = CIBW_CACHE_PATH / f"pyodide-xbuildenv-{pyodide_build_version}" + xbuildenv_cache_path = CIBW_CACHE_PATH / f"pyodide-build-{pyodide_build_version}" pyodide_root = xbuildenv_cache_path / pyodide_version / "xbuildenv" / "pyodide-root" with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"): From 3dbe64ca68143553be63f581f084f2701f237372 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 27 Apr 2025 20:20:00 +0100 Subject: [PATCH 074/109] Remove extra github actions job --- .github/workflows/test.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e9c79cec..553cc2ebe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -169,13 +169,9 @@ jobs: run: uv run --no-sync pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py test-pyodide: - name: Test cibuildwheel building Pyodide wheels (${{ matrix.pyodide-version }} version) + name: Test pyodide needs: lint runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - pyodide-version: ["default", "custom"] timeout-minutes: 180 steps: - uses: actions/checkout@v4 @@ -193,7 +189,6 @@ jobs: uv run --no-sync -m test.test_projects test.test_0_basic.basic_project sample_proj - name: Run a sample build (GitHub Action) - if: matrix.pyodide-version == 'default' uses: ./ with: package-dir: sample_proj @@ -202,7 +197,6 @@ jobs: CIBW_PLATFORM: pyodide - name: Run a sample build (GitHub Action) for an overridden Pyodide version - if: matrix.pyodide-version == 'custom' uses: ./ with: package-dir: sample_proj @@ -212,6 +206,7 @@ jobs: # in cibuildwheel/resources/build-platforms.toml. env: CIBW_PLATFORM: pyodide + CIBW_BUILD: "cp312*" CIBW_PYODIDE_VERSION: "0.27.0a2" - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' From ea6e81f883f907a5426bec953ef92a16744e7e0e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 27 Apr 2025 20:42:59 +0100 Subject: [PATCH 075/109] Fix expectation for test_abi_none --- test/test_abi_variants.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test/test_abi_variants.py b/test/test_abi_variants.py index af4918110..3fbeb4366 100644 --- a/test/test_abi_variants.py +++ b/test/test_abi_variants.py @@ -187,15 +187,20 @@ def test_abi_none(tmp_path, capfd): }, ) - # check that the expected wheels are produced - expected_wheels = utils.expected_wheels("ctypesexample", "1.0.0", python_abi_tags=["py3-none"]) - assert set(actual_wheels) == set(expected_wheels) - - # check that each wheel was built once, and reused captured = capfd.readouterr() - assert "Building wheel..." in captured.out + if utils.platform == "pyodide": - # pyodide has a unique platform tag for each wheel, so they are not reused + # we only selected one python version for pyodide, so we only get one + # wheel + assert set(actual_wheels) == {"ctypesexample-1.0.0-py3-none-pyodide_2024_0_wasm32.whl"} assert "Found previously built wheel" not in captured.out else: + # check that the expected wheels are produced + expected_wheels = utils.expected_wheels( + "ctypesexample", "1.0.0", python_abi_tags=["py3-none"] + ) + assert set(actual_wheels) == set(expected_wheels) + + # check that each wheel was built once, and reused + assert "Building wheel..." in captured.out assert "Found previously built wheel" in captured.out From 1cd3e22745bffbcd14f42c7c0d9ae32d0550ef89 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 27 Apr 2025 20:48:15 +0100 Subject: [PATCH 076/109] Fix the custom_repair_wheel test to actually have clashing names --- test/test_custom_repair_wheel.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/test_custom_repair_wheel.py b/test/test_custom_repair_wheel.py index cd04bc688..5af012653 100644 --- a/test/test_custom_repair_wheel.py +++ b/test/test_custom_repair_wheel.py @@ -16,6 +16,11 @@ wheel = Path(sys.argv[1]) dest_dir = Path(sys.argv[2]) platform = wheel.stem.split("-")[-1] +if platform.startswith("pyodide"): + # for the sake of this test, munge the pyodide platforms into one, it's + # not valid, but it does activate the uniqueness check + platform = "pyodide" + name = f"spam-0.1.0-py2-none-{platform}.whl" dest = dest_dir / name dest_dir.mkdir(parents=True, exist_ok=True) @@ -50,9 +55,4 @@ def test(tmp_path, capfd): else: # We only produced one wheel (perhaps Pyodide) # check that it has the right name - # - # As far as I can tell, this is the only full test coverage for - # CIBW_REPAIR_WHEEL_COMMAND so this is useful even in the case when no - # error is raised - assert "spam-0.1.0-py2-none-pyodide" in captured.out assert result[0].startswith("spam-0.1.0-py2-none-") From 9680dd960dacee61a9b412838936712888e53926 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 28 Apr 2025 18:52:43 +0100 Subject: [PATCH 077/109] Fix pinned version test --- test/test_dependency_versions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index b8d80f3bc..dd7eac26c 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -66,7 +66,10 @@ def test_pinned_versions(tmp_path, python_version, build_frontend_env_nouv): version_no_dot = python_version.replace(".", "") build_environment = {} build_pattern = f"[cp]p{version_no_dot}-*" - constraint_filename = f"constraints-python{version_no_dot}.txt" + if utils.platform == "pyodide": + constraint_filename = f"constraints-pyodide{version_no_dot}.txt" + else: + constraint_filename = f"constraints-python{version_no_dot}.txt" constraint_file = resources.PATH / constraint_filename constraint_versions = get_versions_from_constraint_file(constraint_file) From 60ee8a4bb327a9c3d89d114719ecca6ca859be8e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 8 May 2025 12:38:31 +0100 Subject: [PATCH 078/109] Document test-command limitation --- docs/platforms.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/platforms.md b/docs/platforms.md index 44178f950..fad5c82a5 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -162,7 +162,7 @@ Pyodide is offered as an experimental feature in cibuildwheel. ### System requirements -You need to have a matching host version of Python (unlike all other cibuildwheel platforms). Linux host highly recommended; macOS hosts may work (e.g. invoking `pytest` directly in [`CIBW_TEST_COMMAND`](options.md#test-command) is [currently failing](https://github.com/pyodide/pyodide/issues/4802)) and Windows hosts will not work. +Pyodide builds require a Linux or macOS machine. ### Specifying a pyodide build @@ -172,6 +172,10 @@ You must target pyodide with `--platform pyodide` (or use `--only` on the identi It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` option to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json), which can be viewed more easily by installing `pyodide-build` from PyPI and use `pyodide xbuildenv search --all` to see a compatibility table. +### Running tests + +Currently, it's recommended to run tests using a `python -m` entrypoint, rather than a command line entrypoint, or a shell script. This is because custom entrypoints have some issues in the Pyodide virtual environment. For example, `pytest` may not work as a command line entrypoint, but will work as a `python -m pytest` entrypoint. + ## iOS ### System requirements From 880f66aebdf671be0ae468c31cf56ca3d6ea9a43 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 8 May 2025 12:38:44 +0100 Subject: [PATCH 079/109] Remove pyodide 0.28.0a1 for now --- cibuildwheel/resources/build-platforms.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 2b22d4b05..5ec73ea61 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -176,7 +176,6 @@ python_configurations = [ [pyodide] python_configurations = [ { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.5", node_version = "v22" }, - { identifier = "cp313-pyodide_wasm32", version = "3.13", default_pyodide_version = "0.28.0a1", node_version = "v22" }, ] [ios] From e58bad6b16b1b3d6c7607038e3c54e63ae35d72a Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 8 May 2025 12:45:26 +0100 Subject: [PATCH 080/109] Update constraints files --- .../resources/constraints-pyodide312.txt | 8 +- .../resources/constraints-pyodide313.txt | 112 ------------------ 2 files changed, 4 insertions(+), 116 deletions(-) delete mode 100644 cibuildwheel/resources/constraints-pyodide313.txt diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 81245d61e..699e17578 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -48,11 +48,11 @@ packaging==25.0 # unearth pip==25.1.1 # via -r .nox/update_constraints/tmp/constraints-pyodide.in -platformdirs==4.3.7 +platformdirs==4.3.8 # via # pyodide-build # virtualenv -pydantic==2.11.3 +pydantic==2.11.4 # via # pyodide-build # pyodide-lock @@ -60,7 +60,7 @@ pydantic-core==2.33.2 # via pydantic pygments==2.19.1 # via rich -pyodide-build==0.30.1 +pyodide-build==0.30.2 # via -r .nox/update_constraints/tmp/constraints-pyodide.in pyodide-cli==0.3.0 # via @@ -105,7 +105,7 @@ unearth==0.17.5 # via pyodide-build urllib3==2.4.0 # via requests -virtualenv==20.30.0 +virtualenv==20.31.1 # via # build # pyodide-build diff --git a/cibuildwheel/resources/constraints-pyodide313.txt b/cibuildwheel/resources/constraints-pyodide313.txt deleted file mode 100644 index 995b0046f..000000000 --- a/cibuildwheel/resources/constraints-pyodide313.txt +++ /dev/null @@ -1,112 +0,0 @@ -# This file was autogenerated by uv via the following command: -# nox -s update_constraints -annotated-types==0.7.0 - # via pydantic -anyio==4.9.0 - # via httpx -auditwheel-emscripten==0.1.0 - # via pyodide-build -build==1.2.2.post1 - # via - # -r .nox/update_constraints/tmp/constraints-pyodide.in - # pyodide-build -certifi==2025.4.26 - # via - # httpcore - # httpx - # requests -charset-normalizer==3.4.1 - # via requests -click==8.1.8 - # via typer -distlib==0.3.9 - # via virtualenv -filelock==3.18.0 - # via virtualenv -h11==0.16.0 - # via httpcore -httpcore==1.0.9 - # via httpx -httpx==0.28.1 - # via unearth -idna==3.10 - # via - # anyio - # httpx - # requests -leb128==1.0.8 - # via auditwheel-emscripten -markdown-it-py==3.0.0 - # via rich -mdurl==0.1.2 - # via markdown-it-py -packaging==25.0 - # via - # auditwheel-emscripten - # build - # pyodide-build - # unearth -pip==25.1 - # via -r .nox/update_constraints/tmp/constraints-pyodide.in -platformdirs==4.3.7 - # via - # pyodide-build - # virtualenv -pydantic==2.11.3 - # via - # pyodide-build - # pyodide-lock -pydantic-core==2.33.1 - # via pydantic -pygments==2.19.1 - # via rich -pyodide-build==0.30.1 - # via -r .nox/update_constraints/tmp/constraints-pyodide.in -pyodide-cli==0.3.0 - # via - # auditwheel-emscripten - # pyodide-build -pyodide-lock==0.1.0a7 - # via pyodide-build -pyproject-hooks==1.2.0 - # via build -requests==2.32.3 - # via pyodide-build -resolvelib==1.1.0 - # via pyodide-build -rich==14.0.0 - # via - # pyodide-build - # pyodide-cli - # typer -ruamel-yaml==0.18.10 - # via pyodide-build -shellingham==1.5.4 - # via typer -sniffio==1.3.1 - # via anyio -typer==0.15.2 - # via - # auditwheel-emscripten - # pyodide-build - # pyodide-cli -typing-extensions==4.13.2 - # via - # pydantic - # pydantic-core - # typer - # typing-inspection -typing-inspection==0.4.0 - # via pydantic -unearth==0.17.5 - # via pyodide-build -urllib3==2.4.0 - # via requests -virtualenv==20.30.0 - # via - # build - # pyodide-build -wheel==0.45.1 - # via - # auditwheel-emscripten - # pyodide-build From 2e7d716a9ecfbdac159eed798ebf1b0170e6858d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 8 May 2025 13:24:09 +0100 Subject: [PATCH 081/109] Docs/test fixes post removing pyodide cp313 --- docs/options.md | 2 +- test/test_pyodide.py | 2 -- test/utils.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/options.md b/docs/options.md index b22b01c09..7b87392f3 100644 --- a/docs/options.md +++ b/docs/options.md @@ -57,7 +57,7 @@ When setting the options, you can use shell-style globbing syntax, as per [fnmat | Python 3.10 | cp310-macosx_x86_64
cp310-macosx_universal2
cp310-macosx_arm64 | cp310-win_amd64
cp310-win32
cp310-win_arm64 | cp310-manylinux_x86_64
cp310-manylinux_i686
cp310-musllinux_x86_64
cp310-musllinux_i686 | cp310-manylinux_aarch64
cp310-manylinux_ppc64le
cp310-manylinux_s390x
cp310-manylinux_armv7l
cp310-manylinux_riscv64
cp310-musllinux_aarch64
cp310-musllinux_ppc64le
cp310-musllinux_s390x
cp310-musllinux_armv7l
cp310-musllinux_riscv64 | | | | Python 3.11 | cp311-macosx_x86_64
cp311-macosx_universal2
cp311-macosx_arm64 | cp311-win_amd64
cp311-win32
cp311-win_arm64 | cp311-manylinux_x86_64
cp311-manylinux_i686
cp311-musllinux_x86_64
cp311-musllinux_i686 | cp311-manylinux_aarch64
cp311-manylinux_ppc64le
cp311-manylinux_s390x
cp311-manylinux_armv7l
cp311-manylinux_riscv64
cp311-musllinux_aarch64
cp311-musllinux_ppc64le
cp311-musllinux_s390x
cp311-musllinux_armv7l
cp311-musllinux_riscv64 | | | | Python 3.12 | cp312-macosx_x86_64
cp312-macosx_universal2
cp312-macosx_arm64 | cp312-win_amd64
cp312-win32
cp312-win_arm64 | cp312-manylinux_x86_64
cp312-manylinux_i686
cp312-musllinux_x86_64
cp312-musllinux_i686 | cp312-manylinux_aarch64
cp312-manylinux_ppc64le
cp312-manylinux_s390x
cp312-manylinux_armv7l
cp312-manylinux_riscv64
cp312-musllinux_aarch64
cp312-musllinux_ppc64le
cp312-musllinux_s390x
cp312-musllinux_armv7l
cp312-musllinux_riscv64 | | cp312-pyodide_wasm32 | -| Python 3.13 | cp313-macosx_x86_64
cp313-macosx_universal2
cp313-macosx_arm64 | cp313-win_amd64
cp313-win32
cp313-win_arm64 | cp313-manylinux_x86_64
cp313-manylinux_i686
cp313-musllinux_x86_64
cp313-musllinux_i686 | cp313-manylinux_aarch64
cp313-manylinux_ppc64le
cp313-manylinux_s390x
cp313-manylinux_armv7l
cp313-manylinux_riscv64
cp313-musllinux_aarch64
cp313-musllinux_ppc64le
cp313-musllinux_s390x
cp313-musllinux_armv7l
cp313-musllinux_riscv64 | cp313-ios_arm64_iphoneos
cp313-ios_arm64_iphonesimulator
cp313-ios_x86_64_iphonesimulator | cp313-pyodide_wasm32 | +| Python 3.13 | cp313-macosx_x86_64
cp313-macosx_universal2
cp313-macosx_arm64 | cp313-win_amd64
cp313-win32
cp313-win_arm64 | cp313-manylinux_x86_64
cp313-manylinux_i686
cp313-musllinux_x86_64
cp313-musllinux_i686 | cp313-manylinux_aarch64
cp313-manylinux_ppc64le
cp313-manylinux_s390x
cp313-manylinux_armv7l
cp313-manylinux_riscv64
cp313-musllinux_aarch64
cp313-musllinux_ppc64le
cp313-musllinux_s390x
cp313-musllinux_armv7l
cp313-musllinux_riscv64 | cp313-ios_arm64_iphoneos
cp313-ios_arm64_iphonesimulator
cp313-ios_x86_64_iphonesimulator | | | PyPy3.8 v7.3 | pp38-macosx_x86_64
pp38-macosx_arm64 | pp38-win_amd64 | pp38-manylinux_x86_64
pp38-manylinux_i686 | pp38-manylinux_aarch64 | | | | PyPy3.9 v7.3 | pp39-macosx_x86_64
pp39-macosx_arm64 | pp39-win_amd64 | pp39-manylinux_x86_64
pp39-manylinux_i686 | pp39-manylinux_aarch64 | | | | PyPy3.10 v7.3 | pp310-macosx_x86_64
pp310-macosx_arm64 | pp310-win_amd64 | pp310-manylinux_x86_64
pp310-manylinux_i686 | pp310-manylinux_aarch64 | | | diff --git a/test/test_pyodide.py b/test/test_pyodide.py index b122883d7..48aaff956 100644 --- a/test/test_pyodide.py +++ b/test/test_pyodide.py @@ -72,7 +72,6 @@ def test_pyodide_build(tmp_path, use_pyproject_toml): # check that the expected wheels are produced expected_wheels = [ "spam-0.1.0-cp312-cp312-pyodide_2024_0_wasm32.whl", - "spam-0.1.0-cp313-cp313-pyodide_2025_0_wasm32.whl", ] print("actual_wheels", actual_wheels) @@ -136,7 +135,6 @@ def test_filter(): # check that the expected wheels are produced expected_wheels = [ "spam-0.1.0-cp312-cp312-pyodide_2024_0_wasm32.whl", - "spam-0.1.0-cp313-cp313-pyodide_2025_0_wasm32.whl", ] print("actual_wheels", actual_wheels) print("expected_wheels", expected_wheels) diff --git a/test/utils.py b/test/utils.py index 4c44b6f98..07831d492 100644 --- a/test/utils.py +++ b/test/utils.py @@ -235,7 +235,7 @@ def _expected_wheels( musllinux_versions = ["musllinux_1_2"] if platform == "pyodide" and python_abi_tags is None: - python_abi_tags = ["cp312-cp312", "cp313-cp313"] + python_abi_tags = ["cp312-cp312"] if python_abi_tags is None: python_abi_tags = [ "cp38-cp38", From ef51f83007ed3a60b6c60a80dbff189c8d9350e2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 8 May 2025 15:18:28 +0100 Subject: [PATCH 082/109] Fix ABI test expectation --- test/test_abi_variants.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/test_abi_variants.py b/test/test_abi_variants.py index 572dabb49..f7c2e2e6f 100644 --- a/test/test_abi_variants.py +++ b/test/test_abi_variants.py @@ -48,8 +48,9 @@ def test_abi3(tmp_path): # check that the expected wheels are produced if utils.platform == "pyodide": - # there's only 1 possible configuration for pyodide, cp312 - expected_wheels = utils.expected_wheels("spam", "0.1.0", python_abi_tags=["cp310-abi3"]) + # there's only 1 possible configuration for pyodide, cp312. It builds + # a wheel that is tagged abi3, compatible back to 3.10 + expected_wheels = ["spam-0.1.0-cp310-abi3-pyodide_2024_0_wasm32.whl"] else: expected_wheels = utils.expected_wheels( "spam", From 062003f48cf21ac31e35a5cdd60e48e7ba7575ae Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 10 May 2025 11:37:14 +0100 Subject: [PATCH 083/109] Docs improvements --- docs/options.md | 4 +++- docs/platforms.md | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/options.md b/docs/options.md index 7b87392f3..45c2dcf39 100644 --- a/docs/options.md +++ b/docs/options.md @@ -17,7 +17,9 @@ Default: `auto` - For `linux`, you need [Docker or Podman](#container-engine) running, on Linux, macOS, or Windows. - For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows. - For `ios` you need to be running on macOS, with Xcode and the iOS simulator installed. -- For `pyodide` `python3.12` must be available in `PATH` and you need to be on one of the following runners: x86-64 Linux, arm64 Linux. Intel and Silicon macOS hosts may succeed, though there are known bugs. See [the section on Pyodide](platforms.md#pyodide) for more information. +- For `pyodide`, you need a Linux or macOS machine. + +Check the [platforms](platforms.md) page for more information on platform requirements. This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config. diff --git a/docs/platforms.md b/docs/platforms.md index fad5c82a5..d4e3b6b52 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -3,7 +3,7 @@ title: Platforms --- # Platforms -## Linux +## Linux {: #linux} ### System requirements @@ -31,7 +31,7 @@ Linux wheels are built in [`manylinux`/`musllinux` containers](https://github.co - Alternative Docker images can be specified with the `CIBW_MANYLINUX_*_IMAGE`/`CIBW_MUSLLINUX_*_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [options](options.md#linux-image) for more details. -## macOS +## macOS {: #macos} ### System requirements @@ -138,7 +138,7 @@ Regarding testing, As a workaround, the tag can be fixed before running delocate to repair the wheel. The [`wheel tags`](https://wheel.readthedocs.io/en/stable/reference/wheel_tags.html) command is ideal for this. See [this workflow](https://gist.github.com/anderssonjohan/49f07e33fc5cb2420515a8ac76dc0c95#file-build-pendulum-wheels-yml-L39-L53) for an example usage of `wheel tags`. -## Windows +## Windows {: #windows} ### System requirements @@ -176,7 +176,7 @@ It is also possible to target a specific Pyodide version by setting the `CIBW_PY Currently, it's recommended to run tests using a `python -m` entrypoint, rather than a command line entrypoint, or a shell script. This is because custom entrypoints have some issues in the Pyodide virtual environment. For example, `pytest` may not work as a command line entrypoint, but will work as a `python -m pytest` entrypoint. -## iOS +## iOS {: #ios} ### System requirements From 4533cb18871841a09e46f70a723f143ee27c53e1 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 10 May 2025 11:37:21 +0100 Subject: [PATCH 084/109] Improve some comments --- cibuildwheel/platforms/pyodide.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index c4a05e9cd..eff37a03d 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -149,11 +149,12 @@ def get_xbuildenv_version_info( raise errors.FatalError(msg) -# The xbuildenv version is brought in sync with the pyodide-build version in -# the constraints file, which will always be compatible with the version in -# build-platforms.toml. Hence, this condition really checks only for the case -# where the version is supplied manually through a CIBW_PYODIDE_VERSION -# environment variable and raises an error as appropriate. +# The default pyodide xbuildenv version that's specified in +# build-platforms.toml is compatible with the pyodide-build version that's +# pinned in the bundled constraints file. But if the user changes +# pyodide-version and/or dependency-constraints in the cibuildwheel config, we +# need to check if the xbuildenv version is compatible with the pyodide-build +# version. def validate_pyodide_build_version( xbuildenv_info: PyodideXBuildEnvInfo, pyodide_build_version: str ) -> None: @@ -180,10 +181,9 @@ def validate_pyodide_build_version( def install_xbuildenv(env: dict[str, str], pyodide_build_version: str, pyodide_version: str) -> str: """Install a particular Pyodide xbuildenv version and set a path to the Pyodide root.""" - # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of pyodide-build are - # not guaranteed to match the versions of Pyodide or be in sync with them. Hence, we shall - # specify the pyodide-build version in the root path, which will set up the xbuildenv for - # the requested Pyodide version. + # Since pyodide-build was unvendored from Pyodide v0.27.0, the versions of + # pyodide-build are uncoupled from the versions of Pyodide. So, we specify + # both the pyodide-build version and the Pyodide version in the temp path. xbuildenv_cache_path = CIBW_CACHE_PATH / f"pyodide-build-{pyodide_build_version}" pyodide_root = xbuildenv_cache_path / pyodide_version / "xbuildenv" / "pyodide-root" From 57cdc60bd8f676cc190a962f3002a4a5ebe36f1e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 10 May 2025 11:46:00 +0100 Subject: [PATCH 085/109] Remove logic duplication --- noxfile.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/noxfile.py b/noxfile.py index 6faf7ed62..28e26c241 100755 --- a/noxfile.py +++ b/noxfile.py @@ -15,12 +15,10 @@ See sessions with `nox -l` """ -import json import os import shutil import sys import textwrap -import urllib.request from pathlib import Path import nox @@ -71,6 +69,8 @@ def update_constraints(session: nox.Session) -> None: Update the dependencies inplace. """ + session.install("-e.") + resources = Path("cibuildwheel/resources") if session.venv_backend != "uv": @@ -99,16 +99,15 @@ def update_constraints(session: nox.Session) -> None: resources / "constraints.txt", ) + from cibuildwheel.extra import get_pyodide_xbuildenv_info + + xbuildenv_info = get_pyodide_xbuildenv_info() + build_platforms = nox.project.load_toml(resources / "build-platforms.toml") pyodides = build_platforms["pyodide"]["python_configurations"] for pyodide in pyodides: python_version = ".".join(pyodide["version"].split(".")[:2]) pyodide_version = pyodide["default_pyodide_version"] - - xbuildenv_info_url = "https://raw.githubusercontent.com/pyodide/pyodide/refs/heads/main/pyodide-cross-build-environments.json" - with urllib.request.urlopen(xbuildenv_info_url) as response: - xbuildenv_info = json.loads(response.read().decode("utf-8")) - pyodide_version_xbuildenv_info = xbuildenv_info["releases"][pyodide_version] pyodide_build_min_version = pyodide_version_xbuildenv_info.get("min_pyodide_build_version") From 32d06e6c8e36c3f7565de6fd98898481f4538068 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 10 May 2025 11:58:19 +0100 Subject: [PATCH 086/109] remove pyodide special casing --- test/test_abi_variants.py | 21 +++++++++++---------- test/utils.py | 5 +++++ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/test/test_abi_variants.py b/test/test_abi_variants.py index f7c2e2e6f..fe79c5c01 100644 --- a/test/test_abi_variants.py +++ b/test/test_abi_variants.py @@ -50,7 +50,11 @@ def test_abi3(tmp_path): if utils.platform == "pyodide": # there's only 1 possible configuration for pyodide, cp312. It builds # a wheel that is tagged abi3, compatible back to 3.10 - expected_wheels = ["spam-0.1.0-cp310-abi3-pyodide_2024_0_wasm32.whl"] + expected_wheels = utils.expected_wheels( + "spam", + "0.1.0", + python_abi_tags=["cp310-abi3"], + ) else: expected_wheels = utils.expected_wheels( "spam", @@ -190,20 +194,17 @@ def test_abi_none(tmp_path, capfd): }, ) + expected_wheels = utils.expected_wheels("ctypesexample", "1.0.0", python_abi_tags=["py3-none"]) + # check that the expected wheels are produced + assert set(actual_wheels) == set(expected_wheels) + captured = capfd.readouterr() if utils.platform == "pyodide": - # we only selected one python version for pyodide, so we only get one - # wheel - assert set(actual_wheels) == {"ctypesexample-1.0.0-py3-none-pyodide_2024_0_wasm32.whl"} + # pyodide builds a different platform tag for each python version, so + # wheels are not reused assert "Found previously built wheel" not in captured.out else: - # check that the expected wheels are produced - expected_wheels = utils.expected_wheels( - "ctypesexample", "1.0.0", python_abi_tags=["py3-none"] - ) - assert set(actual_wheels) == set(expected_wheels) - # check that each wheel was built once, and reused assert "Building wheel..." in captured.out assert "Found previously built wheel" in captured.out diff --git a/test/utils.py b/test/utils.py index 07831d492..2c3c67af1 100644 --- a/test/utils.py +++ b/test/utils.py @@ -339,6 +339,11 @@ def _expected_wheels( "cp313-cp313": ["pyodide_2025_0_wasm32"], }.get(python_abi_tag, []) + if not platform_tags: + # for example if the python tag is `none` or `abi3`, all + # platform tags are built with that python tag + platform_tags = ["pyodide_2024_0_wasm32"] + else: msg = f"Unsupported platform {platform!r}" raise Exception(msg) From eb3789cc6e5aaaf650fa546b4f39c6138f689fab Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 10 May 2025 15:13:28 +0100 Subject: [PATCH 087/109] Refactor constraints code to use a utility script, circumventing import issues --- bin/generate_pyodide_constraints.py | 68 +++++++++++++++++++++++++++++ noxfile.py | 42 ++++++------------ 2 files changed, 81 insertions(+), 29 deletions(-) create mode 100755 bin/generate_pyodide_constraints.py diff --git a/bin/generate_pyodide_constraints.py b/bin/generate_pyodide_constraints.py new file mode 100755 index 000000000..b4ba66bc8 --- /dev/null +++ b/bin/generate_pyodide_constraints.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 + +import sys +import textwrap +from pathlib import Path + +import click + +from cibuildwheel.extra import get_pyodide_xbuildenv_info + + +@click.command() +@click.argument( + "pyodide-version", + type=str, +) +@click.option( + "--output-file", + type=click.Path(), + default=None, + help="Output file to write the constraints to. If not provided, the constraints will be printed to stdout.", +) +def generate_pyodide_constraints(pyodide_version: str, output_file: str | None = None) -> None: + """ + Generate constraints for a specific Pyodide version. The constraints are + generated based on the Pyodide version's xbuildenv info, which is retrieved + from the Pyodide repository. + + These constraints should then be 'pinned' using `uv pip compile`. + + Example usage: + + bin/generate_pyodide_constraints.py 0.27.0 + """ + xbuildenv_info = get_pyodide_xbuildenv_info() + try: + pyodide_version_xbuildenv_info = xbuildenv_info["releases"][pyodide_version] + except KeyError as e: + msg = f"Pyodide version {pyodide_version} not found in xbuildenv info. Versions available: {', '.join(xbuildenv_info['releases'].keys())}" + raise click.BadParameter(msg) from e + + pyodide_build_min_version = pyodide_version_xbuildenv_info.get("min_pyodide_build_version") + pyodide_build_max_version = pyodide_version_xbuildenv_info.get("max_pyodide_build_version") + + pyodide_build_specifier_parts: list[str] = [] + + if pyodide_build_min_version: + pyodide_build_specifier_parts.append(f">={pyodide_build_min_version}") + if pyodide_build_max_version: + pyodide_build_specifier_parts.append(f"<={pyodide_build_max_version}") + + pyodide_build_specifier = ",".join(pyodide_build_specifier_parts) + + constraints_txt = textwrap.dedent(f""" + pip + build[virtualenv] + pyodide-build{pyodide_build_specifier} + """) + + if output_file is None: + print(constraints_txt) + else: + Path(output_file).write_text(constraints_txt) + print(f"Constraints written to {output_file}", file=sys.stderr) + + +if __name__ == "__main__": + generate_pyodide_constraints() diff --git a/noxfile.py b/noxfile.py index 28e26c241..b94eff8f8 100755 --- a/noxfile.py +++ b/noxfile.py @@ -18,7 +18,6 @@ import os import shutil import sys -import textwrap from pathlib import Path import nox @@ -69,19 +68,20 @@ def update_constraints(session: nox.Session) -> None: Update the dependencies inplace. """ - session.install("-e.") + session.install("-e.", "click") resources = Path("cibuildwheel/resources") if session.venv_backend != "uv": session.install("uv>=0.1.23") + env = os.environ.copy() + # CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to + # regenerate the constraints files + env["UV_CUSTOM_COMPILE_COMMAND"] = f"nox -s {session.name}" + for minor_version in range(8, 14): python_version = f"3.{minor_version}" - env = os.environ.copy() - # CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to - # regenerate the constraints files - env["UV_CUSTOM_COMPILE_COMMAND"] = f"nox -s {session.name}" output_file = resources / f"constraints-python{python_version.replace('.', '')}.txt" session.run( "uv", @@ -99,36 +99,20 @@ def update_constraints(session: nox.Session) -> None: resources / "constraints.txt", ) - from cibuildwheel.extra import get_pyodide_xbuildenv_info - - xbuildenv_info = get_pyodide_xbuildenv_info() - build_platforms = nox.project.load_toml(resources / "build-platforms.toml") pyodides = build_platforms["pyodide"]["python_configurations"] for pyodide in pyodides: python_version = ".".join(pyodide["version"].split(".")[:2]) pyodide_version = pyodide["default_pyodide_version"] - pyodide_version_xbuildenv_info = xbuildenv_info["releases"][pyodide_version] - - pyodide_build_min_version = pyodide_version_xbuildenv_info.get("min_pyodide_build_version") - pyodide_build_max_version = pyodide_version_xbuildenv_info.get("max_pyodide_build_version") - - pyodide_build_specifier_parts: list[str] = [] - - if pyodide_build_min_version: - pyodide_build_specifier_parts.append(f">={pyodide_build_min_version}") - if pyodide_build_max_version: - pyodide_build_specifier_parts.append(f"<={pyodide_build_max_version}") - - pyodide_build_specifier = ",".join(pyodide_build_specifier_parts) tmp_file = Path(session.create_tmp()) / "constraints-pyodide.in" - tmp_file.write_text( - textwrap.dedent(f""" - pip - build[virtualenv] - pyodide-build{pyodide_build_specifier} - """) + + session.run( + "python", + "bin/generate_pyodide_constraints.py", + "--output-file", + tmp_file, + pyodide_version, ) output_file = resources / f"constraints-pyodide{python_version.replace('.', '')}.txt" From b8fe94a7687c414390ffe721de1834ba43b8b0c5 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 14 May 2025 15:18:15 -0400 Subject: [PATCH 088/109] chore: nicer nox env Signed-off-by: Henry Schreiner --- noxfile.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/noxfile.py b/noxfile.py index 224ffc2f8..638d9d4c6 100755 --- a/noxfile.py +++ b/noxfile.py @@ -15,7 +15,6 @@ See sessions with `nox -l` """ -import os import shutil import sys from pathlib import Path @@ -75,10 +74,9 @@ def update_constraints(session: nox.Session) -> None: if session.venv_backend != "uv": session.install("uv>=0.1.23") - env = os.environ.copy() # CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to # regenerate the constraints files - env["UV_CUSTOM_COMPILE_COMMAND"] = f"nox -s {session.name}" + env = {"UV_CUSTOM_COMPILE_COMMAND": f"nox -s {session.name}"} for minor_version in range(8, 15): python_version = f"3.{minor_version}" From f84443103b96ff9feaf1774bdced12addd5ce0bd Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 14 May 2025 15:28:12 -0400 Subject: [PATCH 089/109] fix: typo in variable name found by copilot Signed-off-by: Henry Schreiner --- cibuildwheel/platforms/pyodide.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index eff37a03d..9841cae13 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -304,9 +304,9 @@ def setup_python( pyodide_build_version=pyodide_build_version, ) - enscripten_version = xbuildenv_info["emscripten"] - log.step(f"Installing Emscripten version: {enscripten_version} ...") - emcc_path = install_emscripten(tmp, enscripten_version) + emscripten_version = xbuildenv_info["emscripten"] + log.step(f"Installing Emscripten version: {emscripten_version} ...") + emcc_path = install_emscripten(tmp, emscripten_version) env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]]) From 5751bb052ed6ff06d66ca0ea3f5f397241f9285e Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 15 May 2025 03:44:40 +0530 Subject: [PATCH 090/109] Apply suggestions from code review --- .github/workflows/test.yml | 2 +- cibuildwheel/extra.py | 2 +- cibuildwheel/platforms/pyodide.py | 4 ++-- cibuildwheel/resources/build-platforms.toml | 2 +- docs/platforms.md | 2 +- test/test_pep518.py | 2 +- unit_test/options_test.py | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 14710ef86..55eeb1ead 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -233,7 +233,7 @@ jobs: env: CIBW_PLATFORM: pyodide CIBW_BUILD: "cp312*" - CIBW_PYODIDE_VERSION: "0.27.0a2" + CIBW_PYODIDE_VERSION: "0.27.6" - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' run: | diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py index b8c7c42fb..e1aabd06a 100644 --- a/cibuildwheel/extra.py +++ b/cibuildwheel/extra.py @@ -87,6 +87,6 @@ class PyodideXBuildEnvInfo(typing.TypedDict): def get_pyodide_xbuildenv_info() -> PyodideXBuildEnvInfo: - xbuildenv_info_url = "https://raw.githubusercontent.com/pyodide/pyodide/refs/heads/main/pyodide-cross-build-environments.json" + xbuildenv_info_url = "https://cdn.statically.io/gh/pyodide/pyodide/refs/heads/main/pyodide-cross-build-environments.json" with urllib.request.urlopen(xbuildenv_info_url) as response: return typing.cast(PyodideXBuildEnvInfo, json.loads(response.read().decode("utf-8"))) diff --git a/cibuildwheel/platforms/pyodide.py b/cibuildwheel/platforms/pyodide.py index 9841cae13..ac26df3e6 100644 --- a/cibuildwheel/platforms/pyodide.py +++ b/cibuildwheel/platforms/pyodide.py @@ -141,9 +141,9 @@ def get_xbuildenv_version_info( return xbuildenv_info msg = unwrap(f""" - Could not find pyodide xbuildenv version {version} in the available + Could not find Pyodide cross-build environment version {version} in the available versions as reported by pyodide-build v{pyodide_build_version}. - Available pyodide xbuild versions are: + Available pyodide xbuildenv versions are: {", ".join(e["version"] for e in xbuildenvs_info if e["compatible"])} """) raise errors.FatalError(msg) diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 736778656..2875f9f24 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -220,7 +220,7 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.5", node_version = "v22" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.6", node_version = "v22" }, ] [ios] diff --git a/docs/platforms.md b/docs/platforms.md index d4e3b6b52..c4ffb9af6 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -170,7 +170,7 @@ You must target pyodide with `--platform pyodide` (or use `--only` on the identi ### Choosing a Pyodide version {: #pyodide-choosing-a-version} -It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` option to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json), which can be viewed more easily by installing `pyodide-build` from PyPI and use `pyodide xbuildenv search --all` to see a compatibility table. +It is also possible to target a specific Pyodide version by setting the `CIBW_PYODIDE_VERSION` option to the desired version. Users are responsible for setting an appropriate Pyodide version according to the `pyodide-build` version. A list is available in Pyodide's [cross-build environments metadata file](https://github.com/pyodide/pyodide/blob/main/pyodide-cross-build-environments.json), which can be viewed more easily by installing `pyodide-build` from PyPI and using `pyodide xbuildenv search --all` to see a compatibility table. ### Running tests diff --git a/test/test_pep518.py b/test/test_pep518.py index e2da77fb4..6aaf6c93c 100644 --- a/test/test_pep518.py +++ b/test/test_pep518.py @@ -61,7 +61,7 @@ def test_pep518(tmp_path, build_frontend_env): assert not (project_dir / "4.1.2").exists() # pypa/build creates a "build" folder & a "*.egg-info" folder for the - # wheel being built, this should be harmless so remove them. Pyodide build + # wheel being built, this should be harmless so remove them. pyodide-build # creates a ".pyodide_build" folder, but this is gitignored with a # .gitignore file inside. contents = [ diff --git a/unit_test/options_test.py b/unit_test/options_test.py index c30c2a478..9e5a1f089 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -34,7 +34,7 @@ environment-pass = ["EXAMPLE_ENV"] -pyodide-version = "0.26.4" +pyodide-version = "0.27.6" [tool.cibuildwheel.macos] test-requires = "else" @@ -91,7 +91,7 @@ def test_options_1(tmp_path, monkeypatch): assert local.manylinux_images["x86_64"] == pinned_x86_64_container_image["manylinux_2_34"] local = options.build_options("cp312-pyodide_wasm32") - assert local.pyodide_version == "0.26.4" + assert local.pyodide_version == "0.27.6" def test_passthrough(tmp_path, monkeypatch): From 9fd41d0400ce740830ceff96e5da7bc638f27449 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 15 May 2025 03:47:44 +0530 Subject: [PATCH 091/109] Some more Pyodide version updates in the docs section --- docs/options.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/options.md b/docs/options.md index d3ee9b959..baf939972 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1241,23 +1241,23 @@ The available Pyodide versions are determined by the version of `pyodide-build` !!! tab examples "Environment variables" ```yaml - # Build Pyodide wheels using Pyodide version 0.26.4 - CIBW_PYODIDE_VERSION: 0.26.4 + # Build Pyodide wheels using Pyodide version 0.27.6 + CIBW_PYODIDE_VERSION: 0.27.6 # Build Pyodide wheels using a specific alpha release - CIBW_PYODIDE_VERSION: 0.27.0a2 + CIBW_PYODIDE_VERSION: 0.28.0a2 ``` !!! tab examples "pyproject.toml" ```toml [tool.cibuildwheel.pyodide] - # Build Pyodide wheels using Pyodide version 0.26.4 + # Build Pyodide wheels using Pyodide version 0.27.6 pyodide-version = "0.26.4" [tool.cibuildwheel.pyodide] # Build Pyodide wheels using a specific alpha release - pyodide-version = "0.27.0a2" + pyodide-version = "0.28.0a2" ``` ## Testing From 7994b0c39fdf287a902d8de52da23008c6314998 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Thu, 15 May 2025 04:04:34 +0530 Subject: [PATCH 092/109] We haven't released Pyodide v0.27.6 yet --- .github/workflows/test.yml | 2 +- cibuildwheel/resources/build-platforms.toml | 2 +- docs/options.md | 6 +++--- unit_test/options_test.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 55eeb1ead..e31304a8d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -233,7 +233,7 @@ jobs: env: CIBW_PLATFORM: pyodide CIBW_BUILD: "cp312*" - CIBW_PYODIDE_VERSION: "0.27.6" + CIBW_PYODIDE_VERSION: "0.27.5" - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' run: | diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 2875f9f24..736778656 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -220,7 +220,7 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.6", node_version = "v22" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.5", node_version = "v22" }, ] [ios] diff --git a/docs/options.md b/docs/options.md index baf939972..d813e1281 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1241,8 +1241,8 @@ The available Pyodide versions are determined by the version of `pyodide-build` !!! tab examples "Environment variables" ```yaml - # Build Pyodide wheels using Pyodide version 0.27.6 - CIBW_PYODIDE_VERSION: 0.27.6 + # Build Pyodide wheels using Pyodide version 0.27.5 + CIBW_PYODIDE_VERSION: 0.27.5 # Build Pyodide wheels using a specific alpha release CIBW_PYODIDE_VERSION: 0.28.0a2 @@ -1252,7 +1252,7 @@ The available Pyodide versions are determined by the version of `pyodide-build` ```toml [tool.cibuildwheel.pyodide] - # Build Pyodide wheels using Pyodide version 0.27.6 + # Build Pyodide wheels using Pyodide version 0.27.5 pyodide-version = "0.26.4" [tool.cibuildwheel.pyodide] diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 9e5a1f089..3351aa615 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -34,7 +34,7 @@ environment-pass = ["EXAMPLE_ENV"] -pyodide-version = "0.27.6" +pyodide-version = "0.27.5" [tool.cibuildwheel.macos] test-requires = "else" @@ -91,7 +91,7 @@ def test_options_1(tmp_path, monkeypatch): assert local.manylinux_images["x86_64"] == pinned_x86_64_container_image["manylinux_2_34"] local = options.build_options("cp312-pyodide_wasm32") - assert local.pyodide_version == "0.27.6" + assert local.pyodide_version == "0.27.5" def test_passthrough(tmp_path, monkeypatch): From 76e8d580bd1eb4730f4ef79ae9d5ffd09039be8e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 15 May 2025 13:51:44 +0100 Subject: [PATCH 093/109] Back to the Github URL for cross-build-environments --- cibuildwheel/extra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py index e1aabd06a..b8c7c42fb 100644 --- a/cibuildwheel/extra.py +++ b/cibuildwheel/extra.py @@ -87,6 +87,6 @@ class PyodideXBuildEnvInfo(typing.TypedDict): def get_pyodide_xbuildenv_info() -> PyodideXBuildEnvInfo: - xbuildenv_info_url = "https://cdn.statically.io/gh/pyodide/pyodide/refs/heads/main/pyodide-cross-build-environments.json" + xbuildenv_info_url = "https://raw.githubusercontent.com/pyodide/pyodide/refs/heads/main/pyodide-cross-build-environments.json" with urllib.request.urlopen(xbuildenv_info_url) as response: return typing.cast(PyodideXBuildEnvInfo, json.loads(response.read().decode("utf-8"))) From 006bb2f1a5fb49ab70505d7faebb44507d0fc733 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 16 May 2025 00:16:46 +0530 Subject: [PATCH 094/109] `pyodide-build`, not `emsdk` for Windows skips Co-Authored-By: Hood Chatham --- test/test_pyodide.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_pyodide.py b/test/test_pyodide.py index 48aaff956..e1dbbd56e 100644 --- a/test/test_pyodide.py +++ b/test/test_pyodide.py @@ -43,7 +43,7 @@ def check_node(): @pytest.mark.parametrize("use_pyproject_toml", [True, False]) def test_pyodide_build(tmp_path, use_pyproject_toml): if sys.platform == "win32": - pytest.skip("emsdk doesn't work correctly on Windows") + pytest.skip("pyodide-build doesn't work correctly on Windows") if use_pyproject_toml: basic_project.files["pyproject.toml"] = textwrap.dedent( @@ -82,7 +82,7 @@ def test_pyodide_build(tmp_path, use_pyproject_toml): def test_pyodide_version_incompatible(tmp_path, capfd): if sys.platform == "win32": - pytest.skip("emsdk doesn't work correctly on Windows") + pytest.skip("pyodide-build doesn't work correctly on Windows") basic_project.generate(tmp_path) @@ -104,7 +104,7 @@ def test_pyodide_version_incompatible(tmp_path, capfd): @pytest.mark.parametrize("expect_failure", [True, False]) def test_pyodide_build_and_test(tmp_path, expect_failure): if sys.platform == "win32": - pytest.skip("emsdk doesn't work correctly on Windows") + pytest.skip("pyodide-build doesn't work correctly on Windows") if expect_failure: basic_project.files["test/spam_test.py"] = textwrap.dedent(r""" From ac64fc46bbbdf6b16a20fe29f4f3f7dd74d8ca30 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 16 May 2025 12:39:57 +0530 Subject: [PATCH 095/109] Move to a separate `_json_request` function Co-Authored-By: Hood Chatham --- cibuildwheel/extra.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py index b8c7c42fb..935b926be 100644 --- a/cibuildwheel/extra.py +++ b/cibuildwheel/extra.py @@ -39,6 +39,11 @@ def dump_python_configurations( return output.getvalue()[:-1] +def _json_request(request, timeout=30): + with urllib.request.urlopen(request, timeout=timeout) as response: + return typing.cast(dict[str, Any], json.load(response)) + + def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: """ Makes a GitHub API request to the given path and returns the JSON response. @@ -50,13 +55,11 @@ def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: } request = urllib.request.Request(api_url, headers=headers) - try: - with urllib.request.urlopen(request, timeout=30) as response: - return typing.cast(dict[str, Any], json.load(response)) - - except (urllib.error.URLError, TimeoutError) as e: - # pylint: disable=E1101 - if max_retries > 0: + for retries in range(max_retries): + try: + return _json_request(request) + except (urllib.error.URLError, TimeoutError) as e: + # pylint: disable=E1101 if ( isinstance(e, urllib.error.HTTPError) and (e.code == 403 or e.code == 429) @@ -68,10 +71,11 @@ def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: time.sleep(wait_time) else: print(f"Retrying GitHub API request due to error: {e}") - return github_api_request(path, max_retries=max_retries - 1) - else: - print(f"GitHub API request failed (Network error: {e}). Check network connection.") - raise e + + if retries == max_retries - 1: + print(f"GitHub API request failed (Network error: {e}). Check network connection.") + raise e + return None class PyodideXBuildEnvRelease(typing.TypedDict): From 5a40c1f1a909d653f240d1a8be337dd6c04ed135 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 16 May 2025 12:45:07 +0530 Subject: [PATCH 096/109] =?UTF-8?q?Rename=20"retries"=20=E2=9E=A1=EF=B8=8F?= =?UTF-8?q?=20"retry=5Fcount"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cibuildwheel/extra.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py index 935b926be..fc3b7ca3d 100644 --- a/cibuildwheel/extra.py +++ b/cibuildwheel/extra.py @@ -55,7 +55,7 @@ def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: } request = urllib.request.Request(api_url, headers=headers) - for retries in range(max_retries): + for retry_count in range(max_retries): try: return _json_request(request) except (urllib.error.URLError, TimeoutError) as e: @@ -72,7 +72,7 @@ def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: else: print(f"Retrying GitHub API request due to error: {e}") - if retries == max_retries - 1: + if retry_count == max_retries - 1: print(f"GitHub API request failed (Network error: {e}). Check network connection.") raise e return None From b94fc00ee4c9d7b240f5e626185517887ddc1928 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 16 May 2025 12:46:05 +0530 Subject: [PATCH 097/109] Add some type hints --- cibuildwheel/extra.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py index fc3b7ca3d..84a750ab7 100644 --- a/cibuildwheel/extra.py +++ b/cibuildwheel/extra.py @@ -39,7 +39,7 @@ def dump_python_configurations( return output.getvalue()[:-1] -def _json_request(request, timeout=30): +def _json_request(request: urllib.request.Request, timeout: int = 30) -> dict[str, Any]: with urllib.request.urlopen(request, timeout=timeout) as response: return typing.cast(dict[str, Any], json.load(response)) @@ -75,7 +75,10 @@ def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: if retry_count == max_retries - 1: print(f"GitHub API request failed (Network error: {e}). Check network connection.") raise e - return None + + # Should never be reached but to keep the type checker happy + msg = "Unexpected execution path in github_api_request" + raise RuntimeError(msg) class PyodideXBuildEnvRelease(typing.TypedDict): From 00137fbe6c8baa04f5f3e1fcf8e28312ecdb8099 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 16 May 2025 13:50:36 +0530 Subject: [PATCH 098/109] Copy env vars before `UV_CUSTOM_COMPILE_COMMAND` --- noxfile.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/noxfile.py b/noxfile.py index 638d9d4c6..a31c147f6 100755 --- a/noxfile.py +++ b/noxfile.py @@ -15,6 +15,7 @@ See sessions with `nox -l` """ +import os import shutil import sys from pathlib import Path @@ -76,7 +77,8 @@ def update_constraints(session: nox.Session) -> None: # CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to # regenerate the constraints files - env = {"UV_CUSTOM_COMPILE_COMMAND": f"nox -s {session.name}"} + env = os.environ.copy() + env["UV_CUSTOM_COMPILE_COMMAND"] = f"nox -s {session.name}" for minor_version in range(8, 15): python_version = f"3.{minor_version}" From 5c8d57b8b23f6b60bd8aba1258aaf5fbb060fbf2 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 16 May 2025 13:54:26 +0530 Subject: [PATCH 099/109] Use `HTTPError.headers.get` instead --- cibuildwheel/extra.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py index 84a750ab7..99560a7f4 100644 --- a/cibuildwheel/extra.py +++ b/cibuildwheel/extra.py @@ -63,10 +63,10 @@ def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]: if ( isinstance(e, urllib.error.HTTPError) and (e.code == 403 or e.code == 429) - and e.get("x-ratelimit-remaining") == "0" + and e.headers.get("x-ratelimit-remaining") == "0" ): - reset_time = int(e.get("x-ratelimit-reset", 0)) - wait_time = max(0, reset_time - int(e.get("date", 0))) + reset_time = int(e.headers.get("x-ratelimit-reset", 0)) + wait_time = max(0, reset_time - int(e.headers.get("date", 0))) print(f"Github rate limit exceeded. Waiting for {wait_time} seconds.") time.sleep(wait_time) else: From 84314e439da2a69d5fd9ffdee654196ad852da72 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 16 May 2025 13:55:46 +0530 Subject: [PATCH 100/109] Remove extra end quote --- cibuildwheel/util/python_build_standalone.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/util/python_build_standalone.py b/cibuildwheel/util/python_build_standalone.py index 3e053bfcd..95721ee54 100644 --- a/cibuildwheel/util/python_build_standalone.py +++ b/cibuildwheel/util/python_build_standalone.py @@ -91,7 +91,7 @@ def _get_pbs_asset( expected_suffix += "-install_only.tar.gz" asset_pattern = f"cpython-{python_version}.*-{expected_suffix}" - print(f"Looking for file with pattern {asset_pattern}'") + print(f"Looking for file with pattern {asset_pattern}") for release in release_data["releases"]: for asset in release["assets"]: From f4c30a40ebe056470d797ed5d60a7f404b13a6ae Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 16 May 2025 17:34:38 +0530 Subject: [PATCH 101/109] Change download tests URL to `https://badssl.com/` Co-Authored-By: Joe Rickerby <1244307+joerick@users.noreply.github.com> --- unit_test/download_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_test/download_test.py b/unit_test/download_test.py index ca54e9dca..72e722a96 100644 --- a/unit_test/download_test.py +++ b/unit_test/download_test.py @@ -5,7 +5,7 @@ from cibuildwheel.util.file import download -DOWNLOAD_URL = "https://raw.githubusercontent.com/pypa/cibuildwheel/v1.6.3/requirements-dev.txt" +DOWNLOAD_URL = "https://badssl.com" def test_download(monkeypatch, tmp_path): From f7e5c00154f1159f4c18bc03a4710d99a567ff26 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Fri, 16 May 2025 17:41:25 +0530 Subject: [PATCH 102/109] Download size changes, too --- unit_test/download_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unit_test/download_test.py b/unit_test/download_test.py index 72e722a96..03dc44042 100644 --- a/unit_test/download_test.py +++ b/unit_test/download_test.py @@ -12,14 +12,14 @@ def test_download(monkeypatch, tmp_path): monkeypatch.delenv("SSL_CERT_FILE", raising=False) dest = tmp_path / "file.txt" download(DOWNLOAD_URL, dest) - assert len(dest.read_bytes()) == 134 + assert len(dest.read_bytes()) == 11673 def test_download_good_ssl_cert_file(monkeypatch, tmp_path): monkeypatch.setenv("SSL_CERT_FILE", certifi.where()) dest = tmp_path / "file.txt" download(DOWNLOAD_URL, dest) - assert len(dest.read_bytes()) == 134 + assert len(dest.read_bytes()) == 11673 def test_download_bad_ssl_cert_file(monkeypatch, tmp_path): From f97d5a8e3d6d52fd93190640db5c4bc2a5c01fdd Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 16 May 2025 17:15:25 +0100 Subject: [PATCH 103/109] Use jsdelivr for Github asset mirroring --- unit_test/download_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unit_test/download_test.py b/unit_test/download_test.py index 03dc44042..4c5f9edc8 100644 --- a/unit_test/download_test.py +++ b/unit_test/download_test.py @@ -5,21 +5,21 @@ from cibuildwheel.util.file import download -DOWNLOAD_URL = "https://badssl.com" +DOWNLOAD_URL = "https://cdn.jsdelivr.net/gh/pypa/cibuildwheel@v1.6.3/requirements-dev.txt" def test_download(monkeypatch, tmp_path): monkeypatch.delenv("SSL_CERT_FILE", raising=False) dest = tmp_path / "file.txt" download(DOWNLOAD_URL, dest) - assert len(dest.read_bytes()) == 11673 + assert len(dest.read_bytes()) == 134 def test_download_good_ssl_cert_file(monkeypatch, tmp_path): monkeypatch.setenv("SSL_CERT_FILE", certifi.where()) dest = tmp_path / "file.txt" download(DOWNLOAD_URL, dest) - assert len(dest.read_bytes()) == 11673 + assert len(dest.read_bytes()) == 134 def test_download_bad_ssl_cert_file(monkeypatch, tmp_path): From f8f6729598d8f3d1372142892df77b627baad94b Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Sat, 17 May 2025 01:34:15 +0530 Subject: [PATCH 104/109] Bump to Pyodide v0.27.6 --- .github/workflows/test.yml | 2 +- cibuildwheel/resources/build-platforms.toml | 2 +- docs/options.md | 8 ++++---- unit_test/options_test.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bc5522d49..684c76cf4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -233,7 +233,7 @@ jobs: env: CIBW_PLATFORM: pyodide CIBW_BUILD: "cp312*" - CIBW_PYODIDE_VERSION: "0.27.5" + CIBW_PYODIDE_VERSION: "0.27.6" - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' run: | diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 736778656..2875f9f24 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -220,7 +220,7 @@ python_configurations = [ [pyodide] python_configurations = [ - { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.5", node_version = "v22" }, + { identifier = "cp312-pyodide_wasm32", version = "3.12", default_pyodide_version = "0.27.6", node_version = "v22" }, ] [ios] diff --git a/docs/options.md b/docs/options.md index dba2071d9..8480d721a 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1242,8 +1242,8 @@ The available Pyodide versions are determined by the version of `pyodide-build` !!! tab examples "Environment variables" ```yaml - # Build Pyodide wheels using Pyodide version 0.27.5 - CIBW_PYODIDE_VERSION: 0.27.5 + # Build Pyodide wheels using Pyodide version 0.27.6 + CIBW_PYODIDE_VERSION: 0.27.6 # Build Pyodide wheels using a specific alpha release CIBW_PYODIDE_VERSION: 0.28.0a2 @@ -1253,8 +1253,8 @@ The available Pyodide versions are determined by the version of `pyodide-build` ```toml [tool.cibuildwheel.pyodide] - # Build Pyodide wheels using Pyodide version 0.27.5 - pyodide-version = "0.26.4" + # Build Pyodide wheels using Pyodide version 0.27.6 + pyodide-version = "0.27.6" [tool.cibuildwheel.pyodide] # Build Pyodide wheels using a specific alpha release diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 3351aa615..bcf380015 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -34,7 +34,7 @@ environment-pass = ["EXAMPLE_ENV"] -pyodide-version = "0.27.5" +pyodide-version = "6" [tool.cibuildwheel.macos] test-requires = "else" @@ -91,7 +91,7 @@ def test_options_1(tmp_path, monkeypatch): assert local.manylinux_images["x86_64"] == pinned_x86_64_container_image["manylinux_2_34"] local = options.build_options("cp312-pyodide_wasm32") - assert local.pyodide_version == "0.27.5" + assert local.pyodide_version == "0.27.6" def test_passthrough(tmp_path, monkeypatch): From 1e03d09f5e1654e5a973fd50b3373539a317e970 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 18 May 2025 15:48:40 +0100 Subject: [PATCH 105/109] Fix unit tests --- unit_test/options_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unit_test/options_test.py b/unit_test/options_test.py index bcf380015..3351aa615 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -34,7 +34,7 @@ environment-pass = ["EXAMPLE_ENV"] -pyodide-version = "6" +pyodide-version = "0.27.5" [tool.cibuildwheel.macos] test-requires = "else" @@ -91,7 +91,7 @@ def test_options_1(tmp_path, monkeypatch): assert local.manylinux_images["x86_64"] == pinned_x86_64_container_image["manylinux_2_34"] local = options.build_options("cp312-pyodide_wasm32") - assert local.pyodide_version == "0.27.6" + assert local.pyodide_version == "0.27.5" def test_passthrough(tmp_path, monkeypatch): From c795a41e4a5a75fe8525024fa69ced0618728605 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Mon, 19 May 2025 20:52:03 +0530 Subject: [PATCH 106/109] Move to pyodide v0.27.6 again --- unit_test/options_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 3351aa615..9e5a1f089 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -34,7 +34,7 @@ environment-pass = ["EXAMPLE_ENV"] -pyodide-version = "0.27.5" +pyodide-version = "0.27.6" [tool.cibuildwheel.macos] test-requires = "else" @@ -91,7 +91,7 @@ def test_options_1(tmp_path, monkeypatch): assert local.manylinux_images["x86_64"] == pinned_x86_64_container_image["manylinux_2_34"] local = options.build_options("cp312-pyodide_wasm32") - assert local.pyodide_version == "0.27.5" + assert local.pyodide_version == "0.27.6" def test_passthrough(tmp_path, monkeypatch): From a9d6434426ac9b4d3d24b092f7909dc55d0506a7 Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 20 May 2025 15:37:03 +0530 Subject: [PATCH 107/109] Bump to pyodide-build 0.30.4 --- cibuildwheel/resources/constraints-pyodide312.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cibuildwheel/resources/constraints-pyodide312.txt b/cibuildwheel/resources/constraints-pyodide312.txt index 3613661ee..e9116769d 100644 --- a/cibuildwheel/resources/constraints-pyodide312.txt +++ b/cibuildwheel/resources/constraints-pyodide312.txt @@ -21,8 +21,6 @@ click==8.1.8 # via # -r .nox/update_constraints/tmp/constraints-pyodide.in # typer -cmake==4.0.2 - # via pyodide-build distlib==0.3.9 # via virtualenv filelock==3.18.0 @@ -64,7 +62,7 @@ pydantic-core==2.33.2 # via pydantic pygments==2.19.1 # via rich -pyodide-build==0.30.2 +pyodide-build==0.30.4 # via -r .nox/update_constraints/tmp/constraints-pyodide.in pyodide-cli==0.3.0 # via From 2396224036400312aaf731d05f7b7ffddcd5ce5a Mon Sep 17 00:00:00 2001 From: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> Date: Tue, 20 May 2025 17:00:40 +0530 Subject: [PATCH 108/109] Use new URL for cross-build environments metadata Co-authored-by: Joe Rickerby --- cibuildwheel/extra.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py index 99560a7f4..ffb568ba1 100644 --- a/cibuildwheel/extra.py +++ b/cibuildwheel/extra.py @@ -94,6 +94,6 @@ class PyodideXBuildEnvInfo(typing.TypedDict): def get_pyodide_xbuildenv_info() -> PyodideXBuildEnvInfo: - xbuildenv_info_url = "https://raw.githubusercontent.com/pyodide/pyodide/refs/heads/main/pyodide-cross-build-environments.json" + xbuildenv_info_url = "https://pyodide.github.io/pyodide/api/pyodide-cross-build-environments.json" with urllib.request.urlopen(xbuildenv_info_url) as response: return typing.cast(PyodideXBuildEnvInfo, json.loads(response.read().decode("utf-8"))) From f9453ed171d8ed225b973fa5ffd1dcf9676ceb5f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 May 2025 11:33:11 +0000 Subject: [PATCH 109/109] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- cibuildwheel/extra.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/extra.py b/cibuildwheel/extra.py index ffb568ba1..e13df6e0b 100644 --- a/cibuildwheel/extra.py +++ b/cibuildwheel/extra.py @@ -94,6 +94,8 @@ class PyodideXBuildEnvInfo(typing.TypedDict): def get_pyodide_xbuildenv_info() -> PyodideXBuildEnvInfo: - xbuildenv_info_url = "https://pyodide.github.io/pyodide/api/pyodide-cross-build-environments.json" + xbuildenv_info_url = ( + "https://pyodide.github.io/pyodide/api/pyodide-cross-build-environments.json" + ) with urllib.request.urlopen(xbuildenv_info_url) as response: return typing.cast(PyodideXBuildEnvInfo, json.loads(response.read().decode("utf-8")))