diff --git a/pyodide_build/build_env.py b/pyodide_build/build_env.py index 1773738e..b269a631 100644 --- a/pyodide_build/build_env.py +++ b/pyodide_build/build_env.py @@ -16,7 +16,6 @@ from pyodide_build import __version__ from pyodide_build.common import search_pyproject_toml, to_bool, xbuildenv_dirname from pyodide_build.config import ConfigManager -from pyodide_build.recipe import load_all_recipes RUST_BUILD_PRELUDE = """ rustup default ${RUST_TOOLCHAIN} @@ -169,25 +168,6 @@ def get_hostsitepackages() -> str: return get_build_flag("HOSTSITEPACKAGES") -@functools.cache -def get_unisolated_packages() -> list[str]: - PYODIDE_ROOT = get_pyodide_root() - - unisolated_file = PYODIDE_ROOT / "unisolated.txt" - if unisolated_file.exists(): - # in xbuild env, read from file - unisolated_packages = unisolated_file.read_text().splitlines() - else: - unisolated_packages = [] - recipe_dir = PYODIDE_ROOT / "packages" - recipes = load_all_recipes(recipe_dir) - for name, config in recipes.items(): - if config.build.cross_build_env: - unisolated_packages.append(name) - - return unisolated_packages - - def platform() -> str: emscripten_version = get_build_flag("PYODIDE_EMSCRIPTEN_VERSION") version = emscripten_version.replace(".", "_") diff --git a/pyodide_build/common.py b/pyodide_build/common.py index c30c0e25..47c2acf0 100644 --- a/pyodide_build/common.py +++ b/pyodide_build/common.py @@ -451,6 +451,41 @@ def to_bool(value: str) -> bool: return value.lower() not in {"", "0", "false", "no", "off"} +def get_host_platform(): + """ + Return a string that identifies the current platform. + Simplified version of get_host_platform in pypa/distlib. + """ + if os.name != "posix": + raise ValueError(f"only posix platforms are supported, got {os.name}") + + # Set for cross builds explicitly + if "_PYTHON_HOST_PLATFORM" in os.environ: + return os.environ["_PYTHON_HOST_PLATFORM"] + + # Try to distinguish various flavours of Unix + (osname, host, release, version, machine) = os.uname() + + # Convert the OS name to lowercase, remove '/' characters, and translate + # spaces (for "Power Macintosh") + osname = osname.lower().replace("/", "") + machine = machine.replace(" ", "_").replace("/", "-") + + if osname[:5] == "linux": + return f"{osname}-{machine}" + elif osname[:6] == "darwin": + import _osx_support + import sysconfig + + osname, release, machine = _osx_support.get_platform_osx( + sysconfig.get_config_vars(), osname, release, machine + ) + else: + raise ValueError(f"unsupported os: {osname}") + + return f"{osname}-{release}-{machine}" + + def download_and_unpack_archive(url: str, path: Path, descr: str) -> None: """ Download the cross-build environment from the given URL and extract it to the given path. diff --git a/pyodide_build/config.py b/pyodide_build/config.py index 89cdc809..932ee57a 100644 --- a/pyodide_build/config.py +++ b/pyodide_build/config.py @@ -179,6 +179,7 @@ def to_env(self) -> dict[str, str]: "build_dependency_index_url": "BUILD_DEPENDENCY_INDEX_URL", # maintainer only "_f2c_fixes_wrapper": "_F2C_FIXES_WRAPPER", + "_build_dependency_fallback_to_pypi": "BUILD_DEPENDENCY_FALLBACK_TO_PYPI", } BUILD_VAR_TO_KEY = {v: k for k, v in BUILD_KEY_TO_VAR.items()} @@ -195,6 +196,7 @@ def to_env(self) -> dict[str, str]: "build_dependency_index_url", # maintainer only "_f2c_fixes_wrapper", + "_build_dependency_fallback_to_pypi", } # Default configuration values. @@ -211,9 +213,10 @@ def to_env(self) -> dict[str, str]: # Other configuration "pyodide_jobs": "1", "skip_emscripten_version_check": "0", - "build_dependency_index_url": "https://pypi.anaconda.org/pyodide/simple", + "build_dependency_index_url": "https://pypi.anaconda.org/pyodide-build/simple", # maintainer only "_f2c_fixes_wrapper": "", + "_build_dependency_fallback_to_pypi": "1", } # Default configs that are computed from other values (often from Makefile.envs) diff --git a/pyodide_build/pypabuild.py b/pyodide_build/pypabuild.py index c72574ea..494c1a87 100644 --- a/pyodide_build/pypabuild.py +++ b/pyodide_build/pypabuild.py @@ -4,9 +4,8 @@ import subprocess as sp import sys import traceback -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Generator, Iterator, Mapping, Sequence from contextlib import contextmanager -from itertools import chain from pathlib import Path from tempfile import TemporaryDirectory from typing import Literal, cast @@ -18,9 +17,7 @@ from pyodide_build import _f2c_fixes, common, pywasmcross from pyodide_build.build_env import ( get_build_flag, - get_hostsitepackages, get_pyversion, - get_unisolated_packages, platform, ) from pyodide_build.io import _BuildSpecExports @@ -54,6 +51,8 @@ "gfortran": "FC", # https://mesonbuild.com/Reference-tables.html#compiler-and-linker-selection-variables } +HOST_ARCH = common.get_host_platform().replace("-", "_").replace(".", "_") + def _gen_runner( cross_build_env: Mapping[str, str], @@ -103,13 +102,6 @@ def symlink_unisolated_packages(env: DefaultIsolatedEnv) -> None: env_site_packages.mkdir(parents=True, exist_ok=True) shutil.copy(sysconfigdata_path, env_site_packages) - host_site_packages = Path(get_hostsitepackages()) - for name in get_unisolated_packages(): - for path in chain( - host_site_packages.glob(f"{name}*"), host_site_packages.glob(f"_{name}*") - ): - (env_site_packages / path.name).unlink(missing_ok=True) - (env_site_packages / path.name).symlink_to(path) def remove_avoided_requirements( @@ -127,7 +119,7 @@ def install_reqs(env: DefaultIsolatedEnv, reqs: set[str]) -> None: env.install( remove_avoided_requirements( reqs, - get_unisolated_packages() + AVOIDED_REQUIREMENTS, + AVOIDED_REQUIREMENTS, ) ) @@ -153,8 +145,33 @@ def _build_in_isolated_env( # first install the build dependencies symlink_unisolated_packages(env) - install_reqs(env, builder.build_system_requires) - installed_requires_for_build = False + index_url_for_cross_build = get_build_flag("BUILD_DEPENDENCY_INDEX_URL") + installed_build_system_requires = ( + False # build dependency for in pyproject.toml + ) + installed_backend_requires = ( + False # dependencies defined by the backend for a given distribution + ) + + with switch_index_url(index_url_for_cross_build): + try: + install_reqs(env, builder.build_system_requires) + installed_build_system_requires = True + except Exception: + pass + + # Disabled for testing + if not installed_build_system_requires: + if common.to_bool(get_build_flag("BUILD_DEPENDENCY_FALLBACK_TO_PYPI")): + print( + f"Failed to install build dependencies from {index_url_for_cross_build}, falling back to default index url" + ) + install_reqs(env, builder.build_system_requires) + else: + print( + f"Failed to install build dependencies from {index_url_for_cross_build}, proceeding the build, but it will fail." + ) + try: build_reqs = builder.get_requires_for_build( distribution, @@ -163,10 +180,10 @@ def _build_in_isolated_env( pass else: install_reqs(env, build_reqs) - installed_requires_for_build = True + installed_backend_requires = True with common.replace_env(build_env): - if not installed_requires_for_build: + if not installed_backend_requires: build_reqs = builder.get_requires_for_build( distribution, config_settings, @@ -242,6 +259,35 @@ def make_command_wrapper_symlinks(symlink_dir: Path) -> dict[str, str]: return env +@contextmanager +def switch_index_url(index_url: str) -> Generator[None, None, None]: + """ + Switch index URL that pip locates the packages. + This function is expected to be used during the process of + installing package build dependencies. + + Parameters + ---------- + index_url: index URL to switch to + """ + + env = { + "PIP_INDEX_URL": index_url, + } + + # For debugging: uncomment the lines below to see the pip error during the package installation + # import build + # build._ctx.VERBOSITY.set(1) + + # def log(msg, *args, **kwargs): + # print(msg, str(kwargs)) + + # build._ctx.LOGGER.set(log) + + with common.replace_env(env) as replaced_env: + yield replaced_env + + @contextmanager def get_build_env( env: dict[str, str], @@ -276,6 +322,7 @@ def get_build_env( env.update(make_command_wrapper_symlinks(symlink_dir)) sysconfig_dir = Path(get_build_flag("TARGETINSTALLDIR")) / "sysconfigdata" + host_pythonpath = Path(get_build_flag("PYTHONPATH")) args["PYTHONPATH"] = sys.path + [str(symlink_dir), str(sysconfig_dir)] args["orig__name__"] = __name__ args["pythoninclude"] = get_build_flag("PYTHONINCLUDE") @@ -290,7 +337,7 @@ def get_build_env( env["_PYTHON_HOST_PLATFORM"] = platform() env["_PYTHON_SYSCONFIGDATA_NAME"] = get_build_flag("SYSCONFIG_NAME") - env["PYTHONPATH"] = str(sysconfig_dir) + env["PYTHONPATH"] = str(sysconfig_dir) + ":" + str(host_pythonpath) env["COMPILER_WRAPPER_DIR"] = str(symlink_dir) yield env diff --git a/pyodide_build/tests/conftest.py b/pyodide_build/tests/conftest.py index 2a504831..ddc3b28c 100644 --- a/pyodide_build/tests/conftest.py +++ b/pyodide_build/tests/conftest.py @@ -62,7 +62,6 @@ def reset_cache(): def _reset(): build_env.get_pyodide_root.cache_clear() build_env.get_build_environment_vars.cache_clear() - build_env.get_unisolated_packages.cache_clear() _reset()