Pyodide improvements: version setting, standalone environments

.github/workflows/test.yml

@@ -195,7 +195,7 @@ jobs:
       run: uv run --no-sync pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py
 
   test-pyodide:
-    name: Test cibuildwheel building Pyodide wheels
+    name: Test pyodide
     needs: lint
     runs-on: ubuntu-24.04
     timeout-minutes: 180
@@ -222,6 +222,19 @@ jobs:
       env:
         CIBW_PLATFORM: pyodide
 
+    - name: Run a sample build (GitHub Action) for an overridden Pyodide version
+      uses: ./
+      with:
+        package-dir: sample_proj
+        output-dir: wheelhouse
+        # In case this breaks at any point in time, switch to using the latest version
+        # available or any other version that is not the same as the default one set
+        # in cibuildwheel/resources/build-platforms.toml.
+      env:
+        CIBW_PLATFORM: pyodide
+        CIBW_BUILD: "cp312*"
+        CIBW_PYODIDE_VERSION: "0.27.6"
+
     - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide'
       run: |
         uv run --no-sync ./bin/run_tests.py

action.yml

@@ -24,11 +24,10 @@ branding:
 runs:
   using: composite
   steps:
-    # Set up the version of Python that supports pyodide
     - uses: actions/setup-python@v5
       id: python
       with:
-        python-version: "3.12"
+        python-version: "3.11 - 3.13"
         update-environment: false
 
     - id: cibw

bin/generate_pyodide_constraints.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+
+import sys
+import textwrap
+from pathlib import Path
+
+import click
+
+from cibuildwheel.extra import get_pyodide_xbuildenv_info
+
+
+@click.command()
+@click.argument(
+    "pyodide-version",
+    type=str,
+)
+@click.option(
+    "--output-file",
+    type=click.Path(),
+    default=None,
+    help="Output file to write the constraints to. If not provided, the constraints will be printed to stdout.",
+)
+def generate_pyodide_constraints(pyodide_version: str, output_file: str | None = None) -> None:
+    """
+    Generate constraints for a specific Pyodide version. The constraints are
+    generated based on the Pyodide version's xbuildenv info, which is retrieved
+    from the Pyodide repository.
+
+    These constraints should then be 'pinned' using `uv pip compile`.
+
+    Example usage:
+
+        bin/generate_pyodide_constraints.py 0.27.0
+    """
+    xbuildenv_info = get_pyodide_xbuildenv_info()
+    try:
+        pyodide_version_xbuildenv_info = xbuildenv_info["releases"][pyodide_version]
+    except KeyError as e:
+        msg = f"Pyodide version {pyodide_version} not found in xbuildenv info. Versions available: {', '.join(xbuildenv_info['releases'].keys())}"
+        raise click.BadParameter(msg) from e
+
+    pyodide_build_min_version = pyodide_version_xbuildenv_info.get("min_pyodide_build_version")
+    pyodide_build_max_version = pyodide_version_xbuildenv_info.get("max_pyodide_build_version")
+
+    pyodide_build_specifier_parts: list[str] = []
+
+    if pyodide_build_min_version:
+        pyodide_build_specifier_parts.append(f">={pyodide_build_min_version}")
+    if pyodide_build_max_version:
+        pyodide_build_specifier_parts.append(f"<={pyodide_build_max_version}")
+
+    pyodide_build_specifier = ",".join(pyodide_build_specifier_parts)
+
+    constraints_txt = textwrap.dedent(f"""
+        pip
+        build[virtualenv]
+        pyodide-build{pyodide_build_specifier}
+        click<8.2
+    """)
+
+    if output_file is None:
+        print(constraints_txt)
+    else:
+        Path(output_file).write_text(constraints_txt)
+        print(f"Constraints written to {output_file}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    generate_pyodide_constraints()

bin/generate_schema.py

@@ -197,6 +197,9 @@
   xbuild-tools:
     description: Binaries on the path that should be included in an isolated cross-build environment
     type: string_array
+  pyodide-version:
+    type: string
+    description: Specify the version of Pyodide to use
   repair-wheel-command:
     description: Execute a shell command to repair each built wheel.
     type: string_array

bin/update_python_build_standalone.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+import json
+
+from cibuildwheel.extra import github_api_request
+from cibuildwheel.util.python_build_standalone import (
+    PythonBuildStandaloneAsset,
+    PythonBuildStandaloneReleaseData,
+)
+from cibuildwheel.util.resources import PYTHON_BUILD_STANDALONE_RELEASES
+
+
+def main() -> None:
+    """
+    This script updates the vendored list of release assets to the latest
+    version of astral-sh/python-build-standalone.
+    """
+
+    # Get the latest release tag from the GitHub API
+    latest_release = github_api_request("repos/astral-sh/python-build-standalone/releases/latest")
+    latest_tag = latest_release["tag_name"]
+
+    # Get the list of assets for the latest release
+    github_assets = github_api_request(
+        f"repos/astral-sh/python-build-standalone/releases/tags/{latest_tag}"
+    )["assets"]
+
+    assets: list[PythonBuildStandaloneAsset] = []
+
+    for github_asset in github_assets:
+        name = github_asset["name"]
+        if not name.endswith("install_only.tar.gz"):
+            continue
+        url = github_asset["browser_download_url"]
+        assets.append({"name": name, "url": url})
+
+    # Write the assets to the JSON file. One day, we might need to support
+    # multiple releases, but for now, we only support the latest one
+    json_file_contents: PythonBuildStandaloneReleaseData = {
+        "releases": [
+            {
+                "tag": latest_tag,
+                "assets": assets,
+            }
+        ]
+    }
+
+    with PYTHON_BUILD_STANDALONE_RELEASES.open("w", encoding="utf-8") as f:
+        json.dump(json_file_contents, f, indent=2)
+        # Add a trailing newline, our pre-commit hook requires it
+        f.write("\n")
+
+    print(
+        f"Updated {PYTHON_BUILD_STANDALONE_RELEASES.name} with {len(assets)} assets for tag {latest_tag}"
+    )
+
+
+if __name__ == "__main__":
+    main()

bin/update_pythons.py

@@ -19,7 +19,7 @@
 from rich.logging import RichHandler
 from rich.syntax import Syntax
 
-from cibuildwheel.extra import dump_python_configurations
+from cibuildwheel.extra import dump_python_configurations, get_pyodide_xbuildenv_info
 
 log = logging.getLogger("cibw")
 
@@ -57,7 +57,14 @@ class ConfigApple(TypedDict):
     url: str
 
 
-AnyConfig = ConfigWinCP | ConfigWinPP | ConfigWinGP | ConfigApple
+class ConfigPyodide(TypedDict):
+    identifier: str
+    version: str
+    default_pyodide_version: str
+    node_version: str
+
+
+AnyConfig = ConfigWinCP | ConfigWinPP | ConfigWinGP | ConfigApple | ConfigPyodide
 
 
 # The following set of "Versions" classes allow the initial call to the APIs to
@@ -347,6 +354,39 @@ def update_version_ios(self, identifier: str, version: Version) -> ConfigApple |
         return None
 
 
+class PyodideVersions:
+    def __init__(self) -> None:
+        xbuildenv_info = get_pyodide_xbuildenv_info()
+        self.releases = xbuildenv_info["releases"]
+
+    def update_version_pyodide(
+        self, identifier: str, version: Version, spec: Specifier, node_version: str
+    ) -> ConfigPyodide | None:
+        # get releases that match the python version
+        releases = [
+            r for r in self.releases.values() if spec.contains(Version(r["python_version"]))
+        ]
+        # sort by version, latest first
+        releases.sort(key=lambda r: Version(r["version"]), reverse=True)
+
+        if not releases:
+            msg = f"Pyodide not found for {spec}!"
+            raise ValueError(msg)
+
+        final_releases = [r for r in releases if not Version(r["version"]).is_prerelease]
+
+        # prefer a final release if available, otherwise use the latest
+        # pre-release
+        release = final_releases[0] if final_releases else releases[0]
+
+        return ConfigPyodide(
+            identifier=identifier,
+            version=str(version),
+            default_pyodide_version=release["version"],
+            node_version=node_version,
+        )
+
+
 # This is a universal interface to all the above Versions classes. Given an
 # identifier, it updates a config dict.
 
@@ -369,6 +409,8 @@ def __init__(self) -> None:
 
         self.graalpy = GraalPyVersions()
 
+        self.pyodide = PyodideVersions()
+
     def update_config(self, config: MutableMapping[str, str]) -> None:
         identifier = config["identifier"]
         version = Version(config["version"])
@@ -407,6 +449,10 @@ def update_config(self, config: MutableMapping[str, str]) -> None:
             config_update = self.windows_arm64.update_version_windows(spec)
         elif "ios" in identifier:
             config_update = self.ios_cpython.update_version_ios(identifier, version)
+        elif "pyodide" in identifier:
+            config_update = self.pyodide.update_version_pyodide(
+                identifier, version, spec, config["node_version"]
+            )
 
         assert config_update is not None, f"{identifier} not found!"
         config.update(**config_update)
@@ -445,6 +491,9 @@ def update_pythons(force: bool, level: str) -> None:
     for config in configs["ios"]["python_configurations"]:
         all_versions.update_config(config)
 
+    for config in configs["pyodide"]["python_configurations"]:
+        all_versions.update_config(config)
+
     result_toml = dump_python_configurations(configs)
 
     rich.print()  # spacer

cibuildwheel/extra.py

@@ -2,9 +2,16 @@
 These are utilities for the `/bin` scripts, not for the `cibuildwheel` program.
 """
 
+import json
+import time
+import typing
+import urllib.error
+import urllib.request
 from collections.abc import Mapping, Sequence
 from io import StringIO
-from typing import Protocol
+from typing import Any, NotRequired, Protocol
+
+from cibuildwheel import __version__ as cibw_version
 
 __all__ = ("Printable", "dump_python_configurations")
 
@@ -30,3 +37,65 @@ def dump_python_configurations(
         output.write("\n")
     # Strip the final newline, to avoid two blank lines at the end.
     return output.getvalue()[:-1]
+
+
+def _json_request(request: urllib.request.Request, timeout: int = 30) -> dict[str, Any]:
+    with urllib.request.urlopen(request, timeout=timeout) as response:
+        return typing.cast(dict[str, Any], json.load(response))
+
+
+def github_api_request(path: str, *, max_retries: int = 3) -> dict[str, Any]:
+    """
+    Makes a GitHub API request to the given path and returns the JSON response.
+    """
+    api_url = f"https://api.github.com/{path}"
+    headers = {
+        "Accept": "application/vnd.github.v3+json",
+        "User-Agent": f"cibuildwheel/{cibw_version}",
+    }
+    request = urllib.request.Request(api_url, headers=headers)
+
+    for retry_count in range(max_retries):
+        try:
+            return _json_request(request)
+        except (urllib.error.URLError, TimeoutError) as e:
+            # pylint: disable=E1101
+            if (
+                isinstance(e, urllib.error.HTTPError)
+                and (e.code == 403 or e.code == 429)
+                and e.headers.get("x-ratelimit-remaining") == "0"
+            ):
+                reset_time = int(e.headers.get("x-ratelimit-reset", 0))
+                wait_time = max(0, reset_time - int(e.headers.get("date", 0)))
+                print(f"Github rate limit exceeded. Waiting for {wait_time} seconds.")
+                time.sleep(wait_time)
+            else:
+                print(f"Retrying GitHub API request due to error: {e}")
+
+            if retry_count == max_retries - 1:
+                print(f"GitHub API request failed (Network error: {e}). Check network connection.")
+                raise e
+
+    # Should never be reached but to keep the type checker happy
+    msg = "Unexpected execution path in github_api_request"
+    raise RuntimeError(msg)
+
+
+class PyodideXBuildEnvRelease(typing.TypedDict):
+    version: str
+    python_version: str
+    emscripten_version: str
+    min_pyodide_build_version: NotRequired[str]
+    max_pyodide_build_version: NotRequired[str]
+
+
+class PyodideXBuildEnvInfo(typing.TypedDict):
+    releases: dict[str, PyodideXBuildEnvRelease]
+
+
+def get_pyodide_xbuildenv_info() -> PyodideXBuildEnvInfo:
+    xbuildenv_info_url = (
+        "https://pyodide.github.io/pyodide/api/pyodide-cross-build-environments.json"
+    )
+    with urllib.request.urlopen(xbuildenv_info_url) as response:
+        return typing.cast(PyodideXBuildEnvInfo, json.loads(response.read().decode("utf-8")))

cibuildwheel/options.py

@@ -111,6 +111,7 @@ class BuildOptions:
     build_frontend: BuildFrontendConfig | None
     config_settings: str
     container_engine: OCIContainerEngineConfig
+    pyodide_version: str | None
 
     @property
     def package_dir(self) -> Path:
@@ -840,6 +841,8 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions:
                 msg = f"Failed to parse container config. {e}"
                 raise errors.ConfigurationError(msg) from e
 
+            pyodide_version = self.reader.get("pyodide-version", env_plat=False)
+
             return BuildOptions(
                 globals=self.globals,
                 test_command=test_command,
@@ -860,6 +863,7 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions:
                 build_frontend=build_frontend,
                 config_settings=config_settings,
                 container_engine=container_engine,
+                pyodide_version=pyodide_version or None,
             )
 
     def check_for_invalid_configuration(self, identifiers: Iterable[str]) -> None: