From a5dc20c558948db900dd0964ac23dcd2787d946c Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 6 Jul 2025 22:13:46 +0900 Subject: [PATCH 01/29] refactor(pypi): move the platform config to MODULE.bazel This splits the configuration that we have for the `rules_python` and the defaults that we set for our users from the actual unit tests where we check that the extension is working correctly. With this we will be able to dog food the API and point users into the `MODULE.bazel` as the example snippet. Work towards #2949 Work towards #2747 --- MODULE.bazel | 56 ++++++++++ python/private/pypi/extension.bzl | 60 +---------- tests/pypi/extension/extension_tests.bzl | 132 ++++++++++------------- 3 files changed, 115 insertions(+), 133 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index b1d8711815..a9b51951b2 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -60,6 +60,62 @@ register_toolchains("@pythons_hub//:all") # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it. pip = use_extension("//python/extensions:pip.bzl", "pip") + +# NOTE @aignas 2025-07-06: we define these platforms to keep backwards compatibility with the +# current `experimental_index_url` implementation. Whilst we stabilize the API this list may be +# updated with a mention in the CHANGELOG. +[ + pip.default( + arch_name = cpu, + config_settings = [ + "@platforms//cpu:{}".format(cpu), + "@platforms//os:linux", + ], + env = {"platform_version": "0"}, + os_name = "linux", + platform = "linux_{}".format(cpu), + ) + for cpu in [ + "x86_64", + "aarch64", + # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the + # `pip.default` extension. i.e. drop the below values - users will have to + # define themselves if they need them. + "arm", + "ppc", + "s390x", + ] +] + +[ + pip.default( + arch_name = cpu, + config_settings = [ + "@platforms//cpu:{}".format(cpu), + "@platforms//os:osx", + ], + # We choose the oldest non-EOL version at the time when we release `rules_python`. + # See https://endoflife.date/macos + env = {"platform_version": "14.0"}, + os_name = "osx", + platform = "osx_{}".format(cpu), + ) + for cpu in [ + "aarch64", + "x86_64", + ] +] + +pip.default( + arch_name = "x86_64", + config_settings = [ + "@platforms//cpu:x86_64", + "@platforms//os:windows", + ], + env = {"platform_version": "0"}, + os_name = "windows", + platform = "windows_x86_64", +) pip.parse( # NOTE @aignas 2024-10-26: We have an integration test that depends on us # being able to build sdists for this hub, so explicitly set this to False. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index a0095f8f15..505458008f 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -393,64 +393,6 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = { else: config["platforms"].pop(platform) -def _create_config(defaults): - if defaults["platforms"]: - return struct(**defaults) - - # NOTE: We have this so that it is easier to maintain unit tests assuming certain - # defaults - for cpu in [ - "x86_64", - "aarch64", - # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the - # `pip.default` extension. i.e. drop the below values - users will have to - # define themselves if they need them. - "arm", - "ppc", - "s390x", - ]: - _configure( - defaults, - arch_name = cpu, - os_name = "linux", - platform = "linux_{}".format(cpu), - config_settings = [ - "@platforms//os:linux", - "@platforms//cpu:{}".format(cpu), - ], - env = {"platform_version": "0"}, - ) - for cpu in [ - "aarch64", - "x86_64", - ]: - _configure( - defaults, - arch_name = cpu, - # We choose the oldest non-EOL version at the time when we release `rules_python`. - # See https://endoflife.date/macos - os_name = "osx", - platform = "osx_{}".format(cpu), - config_settings = [ - "@platforms//os:osx", - "@platforms//cpu:{}".format(cpu), - ], - env = {"platform_version": "14.0"}, - ) - - _configure( - defaults, - arch_name = "x86_64", - os_name = "windows", - platform = "windows_x86_64", - config_settings = [ - "@platforms//os:windows", - "@platforms//cpu:x86_64", - ], - env = {"platform_version": "0"}, - ) - return struct(**defaults) - def parse_modules( module_ctx, _fail = fail, @@ -527,7 +469,7 @@ You cannot use both the additive_build_content and additive_build_content_file a # for what. We could also model the `cp313t` freethreaded as separate platforms. ) - config = _create_config(defaults) + config = struct(**defaults) # TODO @aignas 2025-06-03: Merge override API with the builder? _overriden_whl_set = {} diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 146293ee8d..cf96d4005a 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -56,7 +56,23 @@ def _mod(*, name, default = [], parse = [], override = [], whl_mods = [], is_roo parse = parse, override = override, whl_mods = whl_mods, - default = default, + default = default or [ + _default( + platform = "{}_{}".format(os, cpu), + os_name = os, + arch_name = cpu, + config_settings = [ + "@platforms//os:{}".format(os), + "@platforms//cpu:{}".format(cpu), + ], + ) + for os, cpu in [ + ("linux", "x86_64"), + ("linux", "aarch64"), + ("osx", "aarch64"), + ("windows", "aarch64"), + ] + ], ), is_root = is_root, ) @@ -235,19 +251,18 @@ def _test_simple_multiple_requirements(env): pypi.hub_group_map().contains_exactly({"pypi": {}}) pypi.hub_whl_map().contains_exactly({"pypi": { "simple": { - "pypi_315_simple_osx_aarch64_osx_x86_64": [ + "pypi_315_simple_osx_aarch64": [ whl_config_setting( target_platforms = [ "cp315_osx_aarch64", - "cp315_osx_x86_64", ], version = "3.15", ), ], - "pypi_315_simple_windows_x86_64": [ + "pypi_315_simple_windows_aarch64": [ whl_config_setting( target_platforms = [ - "cp315_windows_x86_64", + "cp315_windows_aarch64", ], version = "3.15", ), @@ -255,12 +270,12 @@ def _test_simple_multiple_requirements(env): }, }}) pypi.whl_libraries().contains_exactly({ - "pypi_315_simple_osx_aarch64_osx_x86_64": { + "pypi_315_simple_osx_aarch64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.2 --hash=sha256:deadb00f", }, - "pypi_315_simple_windows_x86_64": { + "pypi_315_simple_windows_aarch64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "simple==0.0.1 --hash=sha256:deadbeef", @@ -310,24 +325,20 @@ torch==2.4.1 ; platform_machine != 'x86_64' \ pypi.hub_group_map().contains_exactly({"pypi": {}}) pypi.hub_whl_map().contains_exactly({"pypi": { "torch": { - "pypi_315_torch_linux_aarch64_linux_arm_linux_ppc_linux_s390x_osx_aarch64": [ + "pypi_315_torch_linux_aarch64_osx_aarch64_windows_aarch64": [ whl_config_setting( target_platforms = [ "cp315_linux_aarch64", - "cp315_linux_arm", - "cp315_linux_ppc", - "cp315_linux_s390x", "cp315_osx_aarch64", + "cp315_windows_aarch64", ], version = "3.15", ), ], - "pypi_315_torch_linux_x86_64_osx_x86_64_windows_x86_64": [ + "pypi_315_torch_linux_x86_64": [ whl_config_setting( target_platforms = [ "cp315_linux_x86_64", - "cp315_osx_x86_64", - "cp315_windows_x86_64", ], version = "3.15", ), @@ -335,12 +346,12 @@ torch==2.4.1 ; platform_machine != 'x86_64' \ }, }}) pypi.whl_libraries().contains_exactly({ - "pypi_315_torch_linux_aarch64_linux_arm_linux_ppc_linux_s390x_osx_aarch64": { + "pypi_315_torch_linux_aarch64_osx_aarch64_windows_aarch64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1 --hash=sha256:deadbeef", }, - "pypi_315_torch_linux_x86_64_osx_x86_64_windows_x86_64": { + "pypi_315_torch_linux_x86_64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -385,6 +396,23 @@ def _test_torch_experimental_index_url(env): module_ctx = _mock_mctx( _mod( name = "rules_python", + default = [ + _default( + platform = "{}_{}".format(os, cpu), + os_name = os, + arch_name = cpu, + config_settings = [ + "@platforms//os:{}".format(os), + "@platforms//cpu:{}".format(cpu), + ], + ) + for os, cpu in [ + ("linux", "aarch64"), + ("linux", "x86_64"), + ("osx", "aarch64"), + ("windows", "x86_64"), + ] + ], parse = [ _parse( hub_name = "pypi", @@ -444,34 +472,26 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ pypi.hub_whl_map().contains_exactly({"pypi": { "torch": { "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [ - struct( - config_setting = None, + whl_config_setting( filename = "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", - target_platforms = None, version = "3.12", ), ], "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [ - struct( - config_setting = None, + whl_config_setting( filename = "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - target_platforms = None, version = "3.12", ), ], "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [ - struct( - config_setting = None, + whl_config_setting( filename = "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", - target_platforms = None, version = "3.12", ), ], "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [ - struct( - config_setting = None, + whl_config_setting( filename = "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", - target_platforms = None, version = "3.12", ), ], @@ -482,7 +502,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_x86_64", - "osx_x86_64", "windows_x86_64", ], "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", @@ -495,9 +514,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "osx_aarch64", ], "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", @@ -510,7 +526,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_x86_64", - "osx_x86_64", "windows_x86_64", ], "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", @@ -523,9 +538,6 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "osx_aarch64", ], "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", @@ -817,13 +829,9 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "linux_x86_64", "osx_aarch64", - "osx_x86_64", - "windows_x86_64", + "windows_aarch64", ], "extra_pip_args": ["--extra-args-for-sdist-building"], "filename": "any-name.tar.gz", @@ -836,13 +844,9 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "linux_x86_64", "osx_aarch64", - "osx_x86_64", - "windows_x86_64", + "windows_aarch64", ], "filename": "direct_without_sha-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -866,13 +870,9 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "linux_x86_64", "osx_aarch64", - "osx_x86_64", - "windows_x86_64", + "windows_aarch64", ], "filename": "simple-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -884,13 +884,9 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "linux_x86_64", "osx_aarch64", - "osx_x86_64", - "windows_x86_64", + "windows_aarch64", ], "extra_pip_args": ["--extra-args-for-sdist-building"], "filename": "simple-0.0.1.tar.gz", @@ -903,13 +899,9 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "linux_x86_64", "osx_aarch64", - "osx_x86_64", - "windows_x86_64", + "windows_aarch64", ], "filename": "some_pkg-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -921,13 +913,9 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", "linux_x86_64", "osx_aarch64", - "osx_x86_64", - "windows_x86_64", + "windows_aarch64", ], "filename": "some-other-pkg-0.0.1-py3-none-any.whl", "python_interpreter_target": "unit_test_interpreter_target", @@ -995,26 +983,22 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.hub_whl_map().contains_exactly({ "pypi": { "optimum": { - "pypi_315_optimum_linux_aarch64_linux_arm_linux_ppc_linux_s390x_linux_x86_64": [ + "pypi_315_optimum_linux_aarch64_linux_x86_64": [ whl_config_setting( version = "3.15", target_platforms = [ "cp315_linux_aarch64", - "cp315_linux_arm", - "cp315_linux_ppc", - "cp315_linux_s390x", "cp315_linux_x86_64", ], config_setting = None, filename = None, ), ], - "pypi_315_optimum_osx_aarch64_osx_x86_64": [ + "pypi_315_optimum_osx_aarch64": [ whl_config_setting( version = "3.15", target_platforms = [ "cp315_osx_aarch64", - "cp315_osx_x86_64", ], config_setting = None, filename = None, @@ -1025,12 +1009,12 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' }) pypi.whl_libraries().contains_exactly({ - "pypi_315_optimum_linux_aarch64_linux_arm_linux_ppc_linux_s390x_linux_x86_64": { + "pypi_315_optimum_linux_aarch64_linux_x86_64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime-gpu]==1.17.1", }, - "pypi_315_optimum_osx_aarch64_osx_x86_64": { + "pypi_315_optimum_osx_aarch64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime]==1.17.1", From ead607103cf40fbe6bd1d26fa89c976fd0f78c99 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Fri, 4 Jul 2025 09:59:48 +0900 Subject: [PATCH 02/29] fix(pypi): correctly handle custom names in pipstar platforms Before it seems that we were relying on particular names in the pipstar platforms. This ensures that we rely on this less. Whilst at it fix a few typos and improve the formatting of the code. Work towards #2949 Work towards #2747 --- CHANGELOG.md | 2 ++ python/private/pypi/BUILD.bazel | 4 ---- python/private/pypi/evaluate_markers.bzl | 2 +- python/private/pypi/extension.bzl | 14 ++++++++++++-- python/private/pypi/parse_requirements.bzl | 6 +++++- python/private/pypi/pep508_env.bzl | 5 ----- tests/pypi/extension/extension_tests.bzl | 20 +++++++++----------- tests/pypi/pep508/evaluate_tests.bzl | 3 ++- 8 files changed, 31 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 81768af36a..2b57af606e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,6 +82,8 @@ END_UNRELEASED_TEMPLATE * (toolchains) `local_runtime_repo` now checks if the include directory exists before attempting to watch it, fixing issues on macOS with system Python ({gh-issue}`3043`). +* (pypi) The pipstar `defaults` configuration now supports any custom platform + name. {#v0-0-0-added} ### Added diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 2666197786..b098f29e94 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -252,10 +252,6 @@ bzl_library( bzl_library( name = "pep508_env_bzl", srcs = ["pep508_env.bzl"], - deps = [ - ":pep508_platform_bzl", - "//python/private:version_bzl", - ], ) bzl_library( diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index 2b805c33e6..6167cdbc96 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -57,7 +57,7 @@ def evaluate_markers_py(mrctx, *, requirements, python_interpreter, python_inter Args: mrctx: repository_ctx or module_ctx. - requirements: list[str] of the requirement file lines to evaluate. + requirements: {type}`dict[str, list[str]]` of the requirement file lines to evaluate. python_interpreter: str, path to the python_interpreter to use to evaluate the env markers in the given requirements files. It will be only called if the requirements files have env markers. This diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index a0095f8f15..64006bd9db 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -76,7 +76,11 @@ def _platforms(*, python_version, minor_mapping, config): for platform, values in config.platforms.items(): key = "{}_{}".format(abi, platform) - platforms[key] = env(key) | values.env + platforms[key] = env(struct( + abi = abi, + os = values.os_name, + arch = values.arch_name, + )) | values.env return platforms def _create_whl_repos( @@ -348,7 +352,7 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net args["filename"] = src.filename if not enable_pipstar: args["experimental_target_platforms"] = [ - # Get rid of the version fot the target platforms because we are + # Get rid of the version for the target platforms because we are # passing the interpreter any way. Ideally we should search of ways # how to pass the target platforms through the hub repo. p.partition("_")[2] @@ -383,6 +387,12 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = { if key not in _SUPPORTED_PEP508_KEYS: fail("Unsupported key in the PEP508 environment: {}".format(key)) + if not os_name: + fail("'os_name' is required") + + if not arch_name: + fail("'arch_name' is required") + config["platforms"][platform] = struct( name = platform.replace("-", "_").lower(), os_name = os_name, diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index e4a8b90acb..9c610f11d3 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -402,6 +402,10 @@ def _add_dists(*, requirement, index_urls, logger = None): ])) # Filter out the wheels that are incompatible with the target_platforms. - whls = select_whls(whls = whls, want_platforms = requirement.target_platforms, logger = logger) + whls = select_whls( + whls = whls, + want_platforms = requirement.target_platforms, + logger = logger, + ) return whls, sdist diff --git a/python/private/pypi/pep508_env.bzl b/python/private/pypi/pep508_env.bzl index a6efb3c50c..c2d404bc3e 100644 --- a/python/private/pypi/pep508_env.bzl +++ b/python/private/pypi/pep508_env.bzl @@ -15,8 +15,6 @@ """This module is for implementing PEP508 environment definition. """ -load(":pep508_platform.bzl", "platform_from_str") - # See https://stackoverflow.com/a/45125525 platform_machine_aliases = { # These pairs mean the same hardware, but different values may be used @@ -175,9 +173,6 @@ def env(target_platform, *, extra = None): if extra != None: env["extra"] = extra - if type(target_platform) == type(""): - target_platform = platform_from_str(target_platform, python_version = "") - if target_platform.abi: minor_version, _, micro_version = target_platform.abi[3:].partition(".") micro_version = micro_version or "0" diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 146293ee8d..51e11342fc 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -1048,7 +1048,9 @@ def _test_pipstar_platforms(env): name = "rules_python", default = [ _default( - platform = "{}_{}".format(os, cpu), + platform = "my{}_{}".format(os, cpu), + os_name = os, + arch_name = cpu, config_settings = [ "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), @@ -1086,24 +1088,20 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.hub_whl_map().contains_exactly({ "pypi": { "optimum": { - "pypi_315_optimum_linux_x86_64": [ + "pypi_315_optimum_mylinux_x86_64": [ whl_config_setting( version = "3.15", target_platforms = [ - "cp315_linux_x86_64", + "cp315_mylinux_x86_64", ], - config_setting = None, - filename = None, ), ], - "pypi_315_optimum_osx_aarch64": [ + "pypi_315_optimum_myosx_aarch64": [ whl_config_setting( version = "3.15", target_platforms = [ - "cp315_osx_aarch64", + "cp315_myosx_aarch64", ], - config_setting = None, - filename = None, ), ], }, @@ -1111,12 +1109,12 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' }) pypi.whl_libraries().contains_exactly({ - "pypi_315_optimum_linux_x86_64": { + "pypi_315_optimum_mylinux_x86_64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime-gpu]==1.17.1", }, - "pypi_315_optimum_osx_aarch64": { + "pypi_315_optimum_myosx_aarch64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime]==1.17.1", diff --git a/tests/pypi/pep508/evaluate_tests.bzl b/tests/pypi/pep508/evaluate_tests.bzl index 7b6c064b94..cc867f346c 100644 --- a/tests/pypi/pep508/evaluate_tests.bzl +++ b/tests/pypi/pep508/evaluate_tests.bzl @@ -16,6 +16,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility load("//python/private/pypi:pep508_evaluate.bzl", "evaluate", "tokenize") # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_platform.bzl", "platform_from_str") # buildifier: disable=bzl-visibility _tests = [] @@ -262,7 +263,7 @@ def _evaluate_with_aliases(env): }, }.items(): # buildifier: @unsorted-dict-items for input, want in tests.items(): - _check_evaluate(env, input, want, pep508_env(target_platform)) + _check_evaluate(env, input, want, pep508_env(platform_from_str(target_platform, ""))) _tests.append(_evaluate_with_aliases) From 5c29535ff669ac965fc0aeb5578b254ee2c9c2a1 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 16 Jun 2025 22:52:43 +0900 Subject: [PATCH 03/29] fix(pypi): pull fewer wheels Before this we would pull all of the wheels that the user target configuration would be compatible with and that meant that it was not customizable. This also meant that there were a lot of footguns in the configuration where the select statements were not really foolproof. With this PR we select only those sources that need to be for the declared configurations. Freethreaded support should be done by defining extre freethreaded platforms using the new builder API. Work towards #2747 Work towards #2759 Work towards #2849 --- MODULE.bazel | 5 +- examples/bzlmod/MODULE.bazel | 17 ++ python/private/pypi/BUILD.bazel | 13 +- python/private/pypi/evaluate_markers.bzl | 8 +- python/private/pypi/extension.bzl | 189 +++++++++---- python/private/pypi/parse_requirements.bzl | 94 ++++--- python/private/pypi/select_whl.bzl | 130 +++++++++ python/private/pypi/whl_target_platforms.bzl | 132 --------- tests/pypi/extension/extension_tests.bzl | 255 +++++++++-------- .../parse_requirements_tests.bzl | 26 +- tests/pypi/select_whl/BUILD.bazel | 3 + .../select_whl_tests.bzl | 262 ++++++++---------- tests/pypi/whl_target_platforms/BUILD.bazel | 3 - 13 files changed, 620 insertions(+), 517 deletions(-) create mode 100644 python/private/pypi/select_whl.bzl create mode 100644 tests/pypi/select_whl/BUILD.bazel rename tests/pypi/{whl_target_platforms => select_whl}/select_whl_tests.bzl (57%) diff --git a/MODULE.bazel b/MODULE.bazel index b1d8711815..8643fceb7d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -63,6 +63,8 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") pip.parse( # NOTE @aignas 2024-10-26: We have an integration test that depends on us # being able to build sdists for this hub, so explicitly set this to False. + # + # how do we test sdists? Maybe just worth adding a single sdist somewhere? download_only = False, experimental_index_url = "https://pypi.org/simple", hub_name = "rules_python_publish_deps", @@ -155,7 +157,6 @@ dev_pip = use_extension( dev_dependency = True, ) dev_pip.parse( - download_only = True, experimental_index_url = "https://pypi.org/simple", hub_name = "dev_pip", parallel_download = False, @@ -163,14 +164,12 @@ dev_pip.parse( requirements_lock = "//docs:requirements.txt", ) dev_pip.parse( - download_only = True, experimental_index_url = "https://pypi.org/simple", hub_name = "dev_pip", python_version = "3.13", requirements_lock = "//docs:requirements.txt", ) dev_pip.parse( - download_only = True, experimental_index_url = "https://pypi.org/simple", hub_name = "pypiserver", python_version = "3.11", diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 841c096dcf..6b1ee2c351 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -158,6 +158,23 @@ pip.whl_mods( ) use_repo(pip, "whl_mods_hub") +# Because below we are using `windows_aarch64` platform, we have to define various +# properties for it. +pip.default( + arch_name = "aarch64", + config_settings = [ + "@platforms//os:windows", + "@platforms//cpu:aarch64", + ], + env = { + "platform_version": "0", + }, + os_name = "windows", + platform = "windows_aarch64", + platform_tags = ["win_amd64"], + want_abis = [], # default to all ABIs +) + # To fetch pip dependencies, use pip.parse. We can pass in various options, # but typically we pass requirements and the Python version. The Python # version must have been configured by a corresponding `python.toolchain()` diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index b098f29e94..982c64436e 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -120,7 +120,6 @@ bzl_library( ":whl_config_setting_bzl", ":whl_library_bzl", ":whl_repo_name_bzl", - ":whl_target_platforms_bzl", "//python/private:full_version_bzl", "//python/private:normalize_name_bzl", "//python/private:version_bzl", @@ -209,7 +208,7 @@ bzl_library( ":parse_requirements_txt_bzl", ":pypi_repo_utils_bzl", ":requirements_files_by_platform_bzl", - ":whl_target_platforms_bzl", + ":select_whl_bzl", "//python/private:normalize_name_bzl", "//python/private:repo_utils_bzl", ], @@ -359,6 +358,15 @@ bzl_library( ], ) +bzl_library( + name = "select_whl_bzl", + srcs = ["select_whl.bzl"], + deps = [ + ":parse_whl_name_bzl", + "//python/private:version_bzl", + ], +) + bzl_library( name = "simpleapi_download_bzl", srcs = ["simpleapi_download.bzl"], @@ -422,5 +430,4 @@ bzl_library( bzl_library( name = "whl_target_platforms_bzl", srcs = ["whl_target_platforms.bzl"], - deps = [":parse_whl_name_bzl"], ) diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index 6167cdbc96..ee8184ac3b 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -43,11 +43,11 @@ def evaluate_markers(*, requirements, platforms): for req_string, platform_strings in requirements.items(): req = requirement(req_string) for platform_str in platform_strings: - env = platforms.get(platform_str) - if not env: - fail("Please define platform: '{}'".format(platform_str)) + plat = platforms.get(platform_str) + if not plat: + continue - if evaluate(req.marker, env = env): + if evaluate(req.marker, env = plat.env): ret.setdefault(req_string, []).append(platform_str) return ret diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 64006bd9db..a5c1348060 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -72,15 +72,25 @@ def _platforms(*, python_version, minor_mapping, config): version = python_version, minor_mapping = minor_mapping, ) - abi = "cp3{}".format(python_version[2:]) for platform, values in config.platforms.items(): + implementation = values.env["implementation_name"][:2].lower() + abi = "{}3{}".format(implementation, python_version[2:]) key = "{}_{}".format(abi, platform) - platforms[key] = env(struct( + + env_ = env(struct( abi = abi, os = values.os_name, arch = values.arch_name, )) | values.env + platforms[key] = struct( + env = env_, + want_abis = [ + v.format(*python_version.split(".")) + for v in values.want_abis + ], + platform_tags = values.platform_tags, + ) return platforms def _create_whl_repos( @@ -152,6 +162,8 @@ def _create_whl_repos( )) python_interpreter_target = available_interpreters[python_name] + # TODO @aignas 2025-06-29: we should not need the version in the pip_name if + # we are using pipstar and we are downloading the wheel using the downloader pip_name = "{}_{}".format( hub_name, version_label(pip_attr.python_version), @@ -230,6 +242,11 @@ def _create_whl_repos( ), logger = logger, ), + platforms = _platforms( + python_version = pip_attr.python_version, + minor_mapping = minor_mapping, + config = config, + ), extra_pip_args = pip_attr.extra_pip_args, get_index_urls = get_index_urls, evaluate_markers = evaluate_markers, @@ -359,27 +376,19 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net for p in src.target_platforms ] - # Pure python wheels or sdists may need to have a platform here - target_platforms = None - if is_whl and not src.filename.endswith("-any.whl"): - pass - elif is_multiple_versions: - target_platforms = src.target_platforms - return struct( repo_name = whl_repo_name(src.filename, src.sha256), args = args, config_setting = whl_config_setting( version = python_version, - filename = src.filename, - target_platforms = target_platforms, + target_platforms = src.target_platforms, ), ) -def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, override = False): +def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, want_abis, platform_tags, override = False): """Set the value in the config if the value is provided""" config.setdefault("platforms", {}) - if platform: + if platform and (os_name or arch_name or config_settings or platform_tags or env): if not override and config.get("platforms", {}).get(platform): return @@ -393,19 +402,35 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = { if not arch_name: fail("'arch_name' is required") + if platform_tags and "any" not in platform_tags: + # the lowest priority one needs to be the first one + platform_tags = ["any"] + platform_tags + config["platforms"][platform] = struct( name = platform.replace("-", "_").lower(), os_name = os_name, arch_name = arch_name, config_settings = config_settings, - env = env, + want_abis = want_abis or [ + "cp{0}{1}", + "abi3", + "none", + ], + platform_tags = platform_tags, + env = { + # default to this + "implementation_name": "cpython", + } | env, ) else: config["platforms"].pop(platform) -def _create_config(defaults): - if defaults["platforms"]: - return struct(**defaults) +def _set_defaults(defaults): + """Set defaults that rules_python is operating under. + + Because this code is also tested in unit tests, leaving it in MODULE.bazel would be + a little problematic. + """ # NOTE: We have this so that it is easier to maintain unit tests assuming certain # defaults @@ -424,42 +449,62 @@ def _create_config(defaults): arch_name = cpu, os_name = "linux", platform = "linux_{}".format(cpu), + want_abis = [], config_settings = [ "@platforms//os:linux", "@platforms//cpu:{}".format(cpu), ], - env = {"platform_version": "0"}, + platform_tags = [ + "linux_*_{}".format(cpu), + "manylinux_*_{}".format(cpu), + ], + env = { + "platform_version": "0", + }, ) - for cpu in [ - "aarch64", - "x86_64", - ]: + for cpu, platform_tag_cpus in { + "aarch64": ["universal2", "arm64"], + "x86_64": ["universal2", "x86_64"], + }.items(): _configure( defaults, arch_name = cpu, - # We choose the oldest non-EOL version at the time when we release `rules_python`. - # See https://endoflife.date/macos os_name = "osx", platform = "osx_{}".format(cpu), config_settings = [ "@platforms//os:osx", "@platforms//cpu:{}".format(cpu), ], - env = {"platform_version": "14.0"}, + want_abis = [], + platform_tags = [ + "macosx_*_{}".format(suffix) + for suffix in platform_tag_cpus + ], + # We choose the oldest non-EOL version at the time when we release `rules_python`. + # See https://endoflife.date/macos + env = { + "platform_version": "14.0", + }, ) - _configure( - defaults, - arch_name = "x86_64", - os_name = "windows", - platform = "windows_x86_64", - config_settings = [ - "@platforms//os:windows", - "@platforms//cpu:x86_64", - ], - env = {"platform_version": "0"}, - ) - return struct(**defaults) + for cpu, platform_tags in { + "x86_64": ["win_amd64"], + }.items(): + _configure( + defaults, + arch_name = cpu, + os_name = "windows", + platform = "windows_{}".format(cpu), + config_settings = [ + "@platforms//os:windows", + "@platforms//cpu:{}".format(cpu), + ], + want_abis = [], + platform_tags = platform_tags, + env = { + "platform_version": "0", + }, + ) def parse_modules( module_ctx, @@ -515,6 +560,7 @@ You cannot use both the additive_build_content and additive_build_content_file a "enable_pipstar": enable_pipstar, "platforms": {}, } + _set_defaults(defaults) for mod in module_ctx.modules: if not (mod.is_root or mod.name == "rules_python"): continue @@ -527,17 +573,18 @@ You cannot use both the additive_build_content and additive_build_content_file a env = tag.env, os_name = tag.os_name, platform = tag.platform, + platform_tags = tag.platform_tags, + want_abis = tag.want_abis, override = mod.is_root, # TODO @aignas 2025-05-19: add more attr groups: # * for AUTH - the default `netrc` usage could be configured through a common # attribute. # * for index/downloader config. This includes all of those attributes for # overrides, etc. Index overrides per platform could be also used here. - # * for whl selection - selecting preferences of which `platform_tag`s we should use - # for what. We could also model the `cp313t` freethreaded as separate platforms. + # * for whl selection - We could also model the `cp313t` freethreaded as separate platforms. ) - config = _create_config(defaults) + config = struct(**defaults) # TODO @aignas 2025-06-03: Merge override API with the builder? _overriden_whl_set = {} @@ -666,7 +713,14 @@ You cannot use both the additive_build_content and additive_build_content_file a for whl_name, aliases in out.extra_aliases.items(): extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases) exposed_packages.setdefault(hub_name, {}).update(out.exposed_packages) - whl_libraries.update(out.whl_libraries) + for whl_name, lib in out.whl_libraries.items(): + if enable_pipstar: + whl_libraries.setdefault(whl_name, lib) + elif whl_name in lib: + fail("'{}' already in created".format(whl_name)) + else: + # replicate whl_libraries.update(out.whl_libraries) + whl_libraries[whl_name] = lib # TODO @aignas 2024-04-05: how do we support different requirement # cycles for different abis/oses? For now we will need the users to @@ -829,25 +883,6 @@ The list of labels to `config_setting` targets that need to be matched for the p selected. """, ), - "os_name": attr.string( - doc = """\ -The OS name to be used. - -:::{note} -Either this or the appropriate `env` keys should be specified. -::: -""", - ), - "platform": attr.string( - doc = """\ -A platform identifier which will be used as the unique identifier within the extension evaluation. -If you are defining custom platforms in your project and don't want things to clash, use extension -[isolation] feature. - -[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate -""", - ), -} | { "env": attr.string_dict( doc = """\ The values to use for environment markers when evaluating an expression. @@ -873,6 +908,40 @@ This is only used if the {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` is enabled. """, ), # The values for PEP508 env marker evaluation during the lock file parsing + "os_name": attr.string( + doc = """\ +The OS name to be used. + +:::{note} +Either this or the appropriate `env` keys should be specified. +::: +""", + ), + "platform": attr.string( + doc = """\ +A platform identifier which will be used as the unique identifier within the extension evaluation. +If you are defining custom platforms in your project and don't want things to clash, use extension +[isolation] feature. + +[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate +""", + ), + "platform_tags": attr.string_list( + doc = """\ +A list of `platform_tag` matchers so that we can select the best wheel based on the user +preference. Per platform we will select a single wheel and the last match from this list will +take precedence. + +The items in this list can contain a single `*` character that is equivalent to `.*` regex match. +""", + ), + "want_abis": attr.string_list( + doc = """\ +A list of ABIs to select wheels for. The values can be either strings or include template +parameters like `{0}` which will be replaced with python version parts. e.g. `cp{0}{1}` will +result in `cp313` given the full python version is `3.13.5`. +""", + ), } _SUPPORTED_PEP508_KEYS = [ diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 9c610f11d3..6f9ae56fc7 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -31,13 +31,14 @@ load("//python/private:repo_utils.bzl", "repo_utils") load(":index_sources.bzl", "index_sources") load(":parse_requirements_txt.bzl", "parse_requirements_txt") load(":pep508_requirement.bzl", "requirement") -load(":whl_target_platforms.bzl", "select_whls") +load(":select_whl.bzl", "select_whl") def parse_requirements( ctx, *, requirements_by_platform = {}, extra_pip_args = [], + platforms = {}, get_index_urls = None, evaluate_markers = None, extract_url_srcs = True, @@ -46,6 +47,7 @@ def parse_requirements( Args: ctx: A context that has .read function that would read contents from a label. + platforms: The target platform descriptions. requirements_by_platform (label_keyed_string_dict): a way to have different package versions (or different packages) for different os, arch combinations. @@ -88,7 +90,7 @@ def parse_requirements( requirements = {} for file, plats in requirements_by_platform.items(): if logger: - logger.debug(lambda: "Using {} for {}".format(file, plats)) + logger.trace(lambda: "Using {} for {}".format(file, plats)) contents = ctx.read(file) # Parse the requirements file directly in starlark to get the information @@ -161,7 +163,7 @@ def parse_requirements( # VCS package references. env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers) if logger: - logger.debug(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format( + logger.trace(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format( reqs_with_env_markers, env_marker_target_platforms, )) @@ -196,6 +198,7 @@ def parse_requirements( name = name, reqs = reqs, index_urls = index_urls, + platforms = platforms, env_marker_target_platforms = env_marker_target_platforms, extract_url_srcs = extract_url_srcs, logger = logger, @@ -203,7 +206,7 @@ def parse_requirements( ) ret.append(item) if not item.is_exposed and logger: - logger.debug(lambda: "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}".format( + logger.trace(lambda: "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}".format( name, sorted(requirement_target_platforms), sorted(requirements), @@ -219,38 +222,43 @@ def _package_srcs( name, reqs, index_urls, + platforms, logger, env_marker_target_platforms, extract_url_srcs): """A function to return sources for a particular package.""" srcs = {} for r in sorted(reqs.values(), key = lambda r: r.requirement_line): - whls, sdist = _add_dists( - requirement = r, - index_urls = index_urls.get(name), - logger = logger, - ) - target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) - target_platforms = sorted(target_platforms) + extra_pip_args = tuple(r.extra_pip_args) - all_dists = [] + whls - if sdist: - all_dists.append(sdist) + for target_platform in target_platforms: + if platforms and target_platform not in platforms: + fail("The target platform '{}' could not be found in {}".format( + target_platform, + platforms.keys(), + )) - if extract_url_srcs and all_dists: - req_line = r.srcs.requirement - else: - all_dists = [struct( - url = "", - filename = "", - sha256 = "", - yanked = False, - )] - req_line = r.srcs.requirement_line + dist = _add_dists( + requirement = r, + target_platform = platforms.get(target_platform), + index_urls = index_urls.get(name), + logger = logger, + ) + if logger: + logger.debug(lambda: "The whl dist is: {}".format(dist.filename if dist else dist)) + + if extract_url_srcs and dist: + req_line = r.srcs.requirement + else: + dist = struct( + url = "", + filename = "", + sha256 = "", + yanked = False, + ) + req_line = r.srcs.requirement_line - extra_pip_args = tuple(r.extra_pip_args) - for dist in all_dists: key = ( dist.filename, req_line, @@ -269,9 +277,9 @@ def _package_srcs( yanked = dist.yanked, ), ) - for p in target_platforms: - if p not in entry.target_platforms: - entry.target_platforms.append(p) + + if target_platform not in entry.target_platforms: + entry.target_platforms.append(target_platform) return srcs.values() @@ -325,7 +333,7 @@ def host_platform(ctx): repo_utils.get_platforms_cpu_name(ctx), ) -def _add_dists(*, requirement, index_urls, logger = None): +def _add_dists(*, requirement, index_urls, target_platform, logger = None): """Populate dists based on the information from the PyPI index. This function will modify the given requirements_by_platform data structure. @@ -333,6 +341,7 @@ def _add_dists(*, requirement, index_urls, logger = None): Args: requirement: The result of parse_requirements function. index_urls: The result of simpleapi_download. + target_platform: The target_platform information. logger: A logger for printing diagnostic info. """ @@ -342,7 +351,7 @@ def _add_dists(*, requirement, index_urls, logger = None): logger.debug(lambda: "Could not detect the filename from the URL, falling back to pip: {}".format( requirement.srcs.url, )) - return [], None + return None # Handle direct URLs in requirements dist = struct( @@ -353,12 +362,12 @@ def _add_dists(*, requirement, index_urls, logger = None): ) if dist.filename.endswith(".whl"): - return [dist], None + return dist else: - return [], dist + return dist if not index_urls: - return [], None + return None whls = [] sdist = None @@ -401,11 +410,16 @@ def _add_dists(*, requirement, index_urls, logger = None): for reason, dists in yanked.items() ])) - # Filter out the wheels that are incompatible with the target_platforms. - whls = select_whls( + if not target_platform: + # The pipstar platforms are undefined here, so we cannot do any matching + return sdist + + # Select a single wheel that can work on the target_platform + return select_whl( whls = whls, - want_platforms = requirement.target_platforms, + python_version = target_platform.env["python_full_version"], + implementation_name = target_platform.env["implementation_name"], + want_abis = target_platform.want_abis, + platforms = target_platform.platform_tags, logger = logger, - ) - - return whls, sdist + ) or sdist diff --git a/python/private/pypi/select_whl.bzl b/python/private/pypi/select_whl.bzl new file mode 100644 index 0000000000..3d8a90235f --- /dev/null +++ b/python/private/pypi/select_whl.bzl @@ -0,0 +1,130 @@ +"Select a single wheel that fits the parameters of a target platform." + +load("//python/private:version.bzl", "version") +load(":parse_whl_name.bzl", "parse_whl_name") + +# Taken from +# https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#python-tag +_PY_TAGS = { + # "py": Generic Python (does not require implementation-specific features) + "cpython": "cp", + "ironpython": "ip", + "jython": "jy", + "pypy": "pp", +} +_PY = "py" + +def _get_priority(*, tag, values, allow_wildcard = True): + for priority, wp in enumerate(values): + head, sep, tail = wp.partition("*") + if "*" in tail: + fail("only a single '*' can be present in the matcher") + if not allow_wildcard and sep: + fail("'*' is not allowed in the matcher") + + for p in tag.split("."): + if not sep and p == head: + return priority + elif sep and p.startswith(head) and p.endswith(tail): + return priority + + return None + +def select_whl(*, whls, python_version, platforms, want_abis, implementation_name = "cpython", limit = 1, logger = None): + """Select a whl that is the most suitable for the given platform. + + Args: + whls: {type}`list[struct]` a list of candidates which have a `filename` + attribute containing the `whl` filename. + python_version: {type}`str` the target python version. + platforms: {type}`list[str]` the target platform identifiers that may contain + a single `*` character. + implementation_name: {type}`str` the `implementation_name` from the target_platform env. + want_abis: {type}`str` the ABIs that the target_platform is compatible with. + limit: {type}`int` number of wheels to return. Defaults to 1. + logger: {type}`struct` the logger instance. + + Returns: + {type}`list[struct] | struct | None`, a single struct from the `whls` input + argument or `None` if a match is not found. If the `limit` is greater than + one, then we will return a list. + """ + py_version = version.parse(python_version, strict = True) + candidates = {} + implementation = _PY_TAGS.get(implementation_name, implementation_name) + + for whl in whls: + parsed = parse_whl_name(whl.filename) + + if parsed.python_tag.startswith(_PY): + pass + elif not parsed.python_tag.startswith(implementation): + if logger: + logger.debug(lambda: "Discarding the wheel because the implementation '{}' is not compatible with target implementation '{}'".format( + parsed.python_tag, + implementation, + )) + continue + + if parsed.python_tag == "py2.py3": + min_version = "2" + else: + min_version = parsed.python_tag[len(implementation):] + + if len(min_version) > 1: + min_version = "{}.{}".format(min_version[0], min_version[1:]) + + min_whl_py_version = version.parse(min_version, strict = True) + if not version.is_ge(py_version, min_whl_py_version): + if logger: + logger.debug(lambda: "Discarding the wheel because the min version supported based on the wheel ABI tag '{}' ({}) is not compatible with the provided target Python version '{}'".format( + parsed.abi_tag, + min_whl_py_version.string, + py_version.string, + )) + continue + + abi_priority = _get_priority( + tag = parsed.abi_tag, + values = want_abis, + allow_wildcard = False, + ) + if abi_priority == None: + if logger: + logger.debug(lambda: "The abi '{}' does not match given list: {}".format( + parsed.abi_tag, + want_abis, + )) + continue + platform_priority = _get_priority( + tag = parsed.platform_tag, + values = platforms, + ) + if platform_priority == None: + if logger: + logger.debug(lambda: "The platform_tag '{}' does not match given list: {}".format( + parsed.platform_tag, + platforms, + )) + continue + + key = ( + # Ensure that we chose the highest compatible version + parsed.python_tag.startswith(implementation), + platform_priority, + # prefer abi_tags in this order + version.key(min_whl_py_version), + abi_priority, + ) + candidates.setdefault(key, whl) + + if not candidates: + return None + + res = [i[1] for i in sorted(candidates.items())] + if logger: + logger.debug(lambda: "Sorted candidates:\n{}".format( + "\n".join([c.filename for c in res]), + )) + + return res[-1] if limit == 1 else res[-limit:] diff --git a/python/private/pypi/whl_target_platforms.bzl b/python/private/pypi/whl_target_platforms.bzl index 6ea3f120c3..6c3dd5da83 100644 --- a/python/private/pypi/whl_target_platforms.bzl +++ b/python/private/pypi/whl_target_platforms.bzl @@ -16,8 +16,6 @@ A starlark implementation of the wheel platform tag parsing to get the target platform. """ -load(":parse_whl_name.bzl", "parse_whl_name") - # The order of the dictionaries is to keep definitions with their aliases next to each # other _CPU_ALIASES = { @@ -46,136 +44,6 @@ _OS_PREFIXES = { "win": "windows", } # buildifier: disable=unsorted-dict-items -def select_whls(*, whls, want_platforms = [], logger = None): - """Select a subset of wheels suitable for target platforms from a list. - - Args: - whls(list[struct]): A list of candidates which have a `filename` - attribute containing the `whl` filename. - want_platforms(str): The platforms in "{abi}_{os}_{cpu}" or "{os}_{cpu}" format. - logger: A logger for printing diagnostic messages. - - Returns: - A filtered list of items from the `whls` arg where `filename` matches - the selected criteria. If no match is found, an empty list is returned. - """ - if not whls: - return [] - - want_abis = { - "abi3": None, - "none": None, - } - - _want_platforms = {} - version_limit = None - - for p in want_platforms: - if not p.startswith("cp3"): - fail("expected all platforms to start with ABI, but got: {}".format(p)) - - abi, _, os_cpu = p.partition("_") - abi, _, _ = abi.partition(".") - _want_platforms[os_cpu] = None - - # TODO @aignas 2025-04-20: add a test - _want_platforms["{}_{}".format(abi, os_cpu)] = None - - version_limit_candidate = int(abi[3:]) - if not version_limit: - version_limit = version_limit_candidate - if version_limit and version_limit != version_limit_candidate: - fail("Only a single python version is supported for now") - - # For some legacy implementations the wheels may target the `cp3xm` ABI - _want_platforms["{}m_{}".format(abi, os_cpu)] = None - want_abis[abi] = None - want_abis[abi + "m"] = None - - # Also add freethreaded wheels if we find them since we started supporting them - _want_platforms["{}t_{}".format(abi, os_cpu)] = None - want_abis[abi + "t"] = None - - want_platforms = sorted(_want_platforms) - - candidates = {} - for whl in whls: - parsed = parse_whl_name(whl.filename) - - if logger: - logger.trace(lambda: "Deciding whether to use '{}'".format(whl.filename)) - - supported_implementations = {} - whl_version_min = 0 - for tag in parsed.python_tag.split("."): - supported_implementations[tag[:2]] = None - - if tag.startswith("cp3") or tag.startswith("py3"): - version = int(tag[len("..3"):] or 0) - else: - # In this case it should be eithor "cp2" or "py2" and we will default - # to `whl_version_min` = 0 - continue - - if whl_version_min == 0 or version < whl_version_min: - whl_version_min = version - - if not ("cp" in supported_implementations or "py" in supported_implementations): - if logger: - logger.trace(lambda: "Discarding the whl because the whl does not support CPython, whl supported implementations are: {}".format(supported_implementations)) - continue - - if want_abis and parsed.abi_tag not in want_abis: - # Filter out incompatible ABIs - if logger: - logger.trace(lambda: "Discarding the whl because the whl abi did not match") - continue - - if whl_version_min > version_limit: - if logger: - logger.trace(lambda: "Discarding the whl because the whl supported python version is too high") - continue - - compatible = False - if parsed.platform_tag == "any": - compatible = True - else: - for p in whl_target_platforms(parsed.platform_tag, abi_tag = parsed.abi_tag.strip("m") if parsed.abi_tag.startswith("cp") else None): - if p.target_platform in want_platforms: - compatible = True - break - - if not compatible: - if logger: - logger.trace(lambda: "Discarding the whl because the whl does not support the desired platforms: {}".format(want_platforms)) - continue - - for implementation in supported_implementations: - candidates.setdefault( - ( - parsed.abi_tag, - parsed.platform_tag, - ), - {}, - ).setdefault( - ( - # prefer cp implementation - implementation == "cp", - # prefer higher versions - whl_version_min, - # prefer abi3 over none - parsed.abi_tag != "none", - # prefer cpx abi over abi3 - parsed.abi_tag != "abi3", - ), - [], - ).append(whl) - - return [ - candidates[key][sorted(v)[-1]][-1] - for key, v in candidates.items() - ] - def whl_target_platforms(platform_tag, abi_tag = ""): """Parse the wheel abi and platform tags and return (os, cpu) tuples. diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 51e11342fc..30093f3227 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -77,21 +77,22 @@ def _parse_modules(env, enable_pipstar = 0, **kwargs): ) def _default( + *, arch_name = None, config_settings = None, os_name = None, platform = None, + platform_tags = None, env = None, - whl_limit = None, - whl_platforms = None): + want_abis = None): return struct( arch_name = arch_name, os_name = os_name, platform = platform, + platform_tags = platform_tags or [], config_settings = config_settings, env = env or {}, - whl_platforms = whl_platforms, - whl_limit = whl_limit, + want_abis = want_abis or [], ) def _parse( @@ -441,50 +442,60 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ pypi.exposed_packages().contains_exactly({"pypi": ["torch"]}) pypi.hub_group_map().contains_exactly({"pypi": {}}) - pypi.hub_whl_map().contains_exactly({"pypi": { - "torch": { - "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [ - struct( - config_setting = None, - filename = "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", - target_platforms = None, - version = "3.12", - ), - ], - "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [ - struct( - config_setting = None, - filename = "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - target_platforms = None, - version = "3.12", - ), - ], - "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [ - struct( - config_setting = None, - filename = "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", - target_platforms = None, - version = "3.12", - ), - ], - "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [ - struct( - config_setting = None, - filename = "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", - target_platforms = None, - version = "3.12", - ), - ], + pypi.hub_whl_map().contains_exactly({ + "pypi": { + "torch": { + "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [ + whl_config_setting( + target_platforms = ("cp312_linux_x86_64",), + version = "3.12", + ), + ], + "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [ + whl_config_setting( + version = "3.12", + target_platforms = ("cp312_linux_aarch64",), + ), + ], + "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [ + whl_config_setting( + target_platforms = ("cp312_windows_x86_64",), + version = "3.12", + ), + ], + "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [ + whl_config_setting( + target_platforms = ("cp312_osx_aarch64",), + version = "3.12", + ), + ], + # we are falling back onto pip to install/fail on this + # TODO @aignas 2025-07-04: maybe we should just exclude the results for + # this? + "pypi_312_torch_linux_arm_linux_ppc_linux_s390x": [ + whl_config_setting( + version = "3.12", + target_platforms = ( + "cp312_linux_arm", + "cp312_linux_ppc", + "cp312_linux_s390x", + ), + ), + ], + # we are falling back onto pip to install/fail on this + "pypi_312_torch_osx_x86_64": [ + whl_config_setting( + target_platforms = ("cp312_osx_x86_64",), + version = "3.12", + ), + ], + }, }, - }}) + }) pypi.whl_libraries().contains_exactly({ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_x86_64", - "osx_x86_64", - "windows_x86_64", - ], + "experimental_target_platforms": ["linux_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -493,13 +504,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", - "osx_aarch64", - ], + "experimental_target_platforms": ["linux_aarch64"], "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -508,11 +513,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_x86_64", - "osx_x86_64", - "windows_x86_64", - ], + "experimental_target_platforms": ["windows_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -521,19 +522,23 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", - "osx_aarch64", - ], + "experimental_target_platforms": ["osx_aarch64"], "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", "sha256": "72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d", "urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"], }, + "pypi_312_torch_linux_arm_linux_ppc_linux_s390x": { + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "torch==2.4.1 --hash=sha256:1495132f30f722af1a091950088baea383fe39903db06b20e6936fd99402803e --hash=sha256:30be2844d0c939161a11073bfbaf645f1c7cb43f62f46cc6e4df1c119fb2a798 --hash=sha256:36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a --hash=sha256:56ad2a760b7a7882725a1eebf5657abbb3b5144eb26bcb47b52059357463c548 --hash=sha256:5fc1d4d7ed265ef853579caf272686d1ed87cebdcd04f2a498f800ffc53dab71 --hash=sha256:72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d --hash=sha256:a38de2803ee6050309aac032676536c3d3b6a9804248537e38e098d0e14817ec --hash=sha256:d36a8ef100f5bff3e9c3cea934b9e0d7ea277cb8210c7152d34a9a6c5830eadd --hash=sha256:ddddbd8b066e743934a4200b3d54267a46db02106876d21cf31f7da7a96f98ea --hash=sha256:fa27b048d32198cda6e9cff0bf768e8683d98743903b7e5d2b1f5098ded1d343", + }, + "pypi_312_torch_osx_x86_64": { + "dep_template": "@pypi//{name}:{target}", + "python_interpreter_target": "unit_test_interpreter_target", + "requirement": "torch==2.4.1+cpu --hash=sha256:0c0a7cc4f7c74ff024d5a5e21230a01289b65346b27a626f6c815d94b4b8c955 --hash=sha256:1dd062d296fb78aa7cfab8690bf03704995a821b5ef69cfc807af5c0831b4202 --hash=sha256:2b03e20f37557d211d14e3fb3f71709325336402db132a1e0dd8b47392185baf --hash=sha256:330e780f478707478f797fdc82c2a96e9b8c5f60b6f1f57bb6ad1dd5b1e7e97e --hash=sha256:3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97 --hash=sha256:3c99506980a2fb4b634008ccb758f42dd82f93ae2830c1e41f64536e310bf562 --hash=sha256:76a6fe7b10491b650c630bc9ae328df40f79a948296b41d3b087b29a8a63cbad --hash=sha256:833490a28ac156762ed6adaa7c695879564fa2fd0dc51bcf3fdb2c7b47dc55e6 --hash=sha256:8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364 --hash=sha256:c4f2c3c026e876d4dad7629170ec14fff48c076d6c2ae0e354ab3fdc09024f00", + }, }) pypi.whl_mods().contains_exactly({}) @@ -734,78 +739,99 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "pypi": { "direct_sdist_without_sha": { "pypi_315_any_name": [ - struct( - config_setting = None, - filename = "any-name.tar.gz", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_arm", + "cp315_linux_ppc", + "cp315_linux_s390x", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_osx_x86_64", + "cp315_windows_x86_64", + ), version = "3.15", ), ], }, "direct_without_sha": { "pypi_315_direct_without_sha_0_0_1_py3_none_any": [ - struct( - config_setting = None, - filename = "direct_without_sha-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_arm", + "cp315_linux_ppc", + "cp315_linux_s390x", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_osx_x86_64", + "cp315_windows_x86_64", + ), version = "3.15", ), ], }, "git_dep": { "pypi_315_git_dep": [ - struct( - config_setting = None, - filename = None, - target_platforms = None, + whl_config_setting( version = "3.15", ), ], }, "pip_fallback": { "pypi_315_pip_fallback": [ - struct( - config_setting = None, - filename = None, - target_platforms = None, + whl_config_setting( version = "3.15", ), ], }, "simple": { "pypi_315_simple_py3_none_any_deadb00f": [ - struct( - config_setting = None, - filename = "simple-0.0.1-py3-none-any.whl", - target_platforms = None, - version = "3.15", - ), - ], - "pypi_315_simple_sdist_deadbeef": [ - struct( - config_setting = None, - filename = "simple-0.0.1.tar.gz", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_arm", + "cp315_linux_ppc", + "cp315_linux_s390x", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_osx_x86_64", + "cp315_windows_x86_64", + ), version = "3.15", ), ], }, "some_other_pkg": { "pypi_315_some_py3_none_any_deadb33f": [ - struct( - config_setting = None, - filename = "some-other-pkg-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_arm", + "cp315_linux_ppc", + "cp315_linux_s390x", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_osx_x86_64", + "cp315_windows_x86_64", + ), version = "3.15", ), ], }, "some_pkg": { "pypi_315_some_pkg_py3_none_any_deadbaaf": [ - struct( - config_setting = None, - filename = "some_pkg-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_arm", + "cp315_linux_ppc", + "cp315_linux_s390x", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_osx_x86_64", + "cp315_windows_x86_64", + ), version = "3.15", ), ], @@ -880,25 +906,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "sha256": "deadb00f", "urls": ["example2.org"], }, - "pypi_315_simple_sdist_deadbeef": { - "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_arm", - "linux_ppc", - "linux_s390x", - "linux_x86_64", - "osx_aarch64", - "osx_x86_64", - "windows_x86_64", - ], - "extra_pip_args": ["--extra-args-for-sdist-building"], - "filename": "simple-0.0.1.tar.gz", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.1", - "sha256": "deadbeef", - "urls": ["example.org"], - }, "pypi_315_some_pkg_py3_none_any_deadbaaf": { "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ @@ -1060,6 +1067,18 @@ def _test_pipstar_platforms(env): ("linux", "x86_64"), ("osx", "aarch64"), ] + ] + [ + _default(platform = name) + for name in [ + "linux_x86_64", + "linux_aarch64", + "linux_arm", + "linux_ppc", + "linux_s390x", + "osx_x86_64", + "osx_aarch64", + "windows_x86_64", + ] ], parse = [ _parse( diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 82fdd0a051..d862d0ebde 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -16,6 +16,8 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private/pypi:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_platform.bzl", "platform_from_str") # buildifier: disable=bzl-visibility def _mock_ctx(): testdata = { @@ -522,6 +524,18 @@ def _test_overlapping_shas_with_index_results(env): "requirements_linux": ["cp39_linux_x86_64"], "requirements_osx": ["cp39_osx_x86_64"], }, + platforms = { + "cp39_linux_x86_64": struct( + platform_tags = ["any"], + env = pep508_env(platform_from_str("cp39_linux_x86_64", "")), + want_abis = ["none"], + ), + "cp39_osx_x86_64": struct( + platform_tags = ["macosx_*"], + env = pep508_env(platform_from_str("cp39_linux_x86_64", "")), + want_abis = ["none"], + ), + }, get_index_urls = lambda _, __: { "foo": struct( sdists = { @@ -563,20 +577,10 @@ def _test_overlapping_shas_with_index_results(env): filename = "foo-0.0.1-py3-none-any.whl", requirement_line = "foo==0.0.3", sha256 = "deadbaaf", - target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"], + target_platforms = ["cp39_linux_x86_64"], url = "super2", yanked = False, ), - struct( - distribution = "foo", - extra_pip_args = [], - filename = "foo-0.0.1.tar.gz", - requirement_line = "foo==0.0.3", - sha256 = "5d15t", - target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"], - url = "sdist", - yanked = False, - ), struct( distribution = "foo", extra_pip_args = [], diff --git a/tests/pypi/select_whl/BUILD.bazel b/tests/pypi/select_whl/BUILD.bazel new file mode 100644 index 0000000000..0ad8cba0cd --- /dev/null +++ b/tests/pypi/select_whl/BUILD.bazel @@ -0,0 +1,3 @@ +load(":select_whl_tests.bzl", "select_whl_test_suite") + +select_whl_test_suite(name = "select_whl_tests") diff --git a/tests/pypi/whl_target_platforms/select_whl_tests.bzl b/tests/pypi/select_whl/select_whl_tests.bzl similarity index 57% rename from tests/pypi/whl_target_platforms/select_whl_tests.bzl rename to tests/pypi/select_whl/select_whl_tests.bzl index 1674ac5ef2..57ff39e76a 100644 --- a/tests/pypi/whl_target_platforms/select_whl_tests.bzl +++ b/tests/pypi/select_whl/select_whl_tests.bzl @@ -1,22 +1,8 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - "" load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "REPO_VERBOSITY_ENV_VAR", "repo_utils") # buildifier: disable=bzl-visibility -load("//python/private/pypi:whl_target_platforms.bzl", "select_whls") # buildifier: disable=bzl-visibility +load("//python/private/pypi:select_whl.bzl", "select_whl") # buildifier: disable=bzl-visibility WHL_LIST = [ "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", @@ -70,6 +56,9 @@ WHL_LIST = [ "pkg-0.0.1-py310-abi3-any.whl", "pkg-0.0.1-py3-abi3-any.whl", "pkg-0.0.1-py3-none-any.whl", + # Extra examples that should be discarded + "pkg-0.0.1-py27-cp27mu-win_amd64.whl", + "pkg-0.0.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", ] def _match(env, got, *want_filenames): @@ -77,15 +66,16 @@ def _match(env, got, *want_filenames): env.expect.that_collection(got).has_size(len(want_filenames)) return + got = [g for g in got if g] got_filenames = [g.filename for g in got] - env.expect.that_collection(got_filenames).contains_exactly(want_filenames) + env.expect.that_collection(got_filenames).contains_exactly(want_filenames).in_order() if got: # Check that we pass the original structs env.expect.that_str(got[0].other).equals("dummy") -def _select_whls(whls, debug = False, **kwargs): - return select_whls( +def _select_whl(whls, debug = False, **kwargs): + return select_whl( whls = [ struct( filename = f, @@ -107,203 +97,189 @@ def _select_whls(whls, debug = False, **kwargs): _tests = [] def _test_simplest(env): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ], - want_platforms = ["cp30_ignored"], + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py3-none-any.whl", + ] + + got = _select_whl( + whls = whls, + platforms = ["any"], + want_abis = ["abi3"], + python_version = "3.0", ) _match( env, - got, + [got], "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", ) _tests.append(_test_simplest) def _test_select_by_supported_py_version(env): + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py311-abi3-any.whl", + ] + for minor_version, match in { 8: "pkg-0.0.1-py3-abi3-any.whl", 11: "pkg-0.0.1-py311-abi3-any.whl", }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py311-abi3-any.whl", - ], - want_platforms = ["cp3{}_ignored".format(minor_version)], + got = _select_whl( + whls = whls, + platforms = ["any"], + want_abis = ["abi3"], + python_version = "3.{}".format(minor_version), ) - _match(env, got, match) + _match(env, [got], match) _tests.append(_test_select_by_supported_py_version) def _test_select_by_supported_cp_version(env): + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py311-abi3-any.whl", + "pkg-0.0.1-cp311-abi3-any.whl", + ] + for minor_version, match in { 11: "pkg-0.0.1-cp311-abi3-any.whl", 8: "pkg-0.0.1-py3-abi3-any.whl", }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py311-abi3-any.whl", - "pkg-0.0.1-cp311-abi3-any.whl", - ], - want_platforms = ["cp3{}_ignored".format(minor_version)], + got = _select_whl( + whls = whls, + platforms = ["any"], + want_abis = ["abi3"], + python_version = "3.{}".format(minor_version), ) - _match(env, got, match) + _match(env, [got], match) _tests.append(_test_select_by_supported_cp_version) def _test_supported_cp_version_manylinux(env): + whls = [ + "pkg-0.0.1-py2.py3-none-manylinux_x86_64.whl", + "pkg-0.0.1-py3-none-manylinux_x86_64.whl", + "pkg-0.0.1-py311-none-manylinux_x86_64.whl", + "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", + ] + for minor_version, match in { 8: "pkg-0.0.1-py3-none-manylinux_x86_64.whl", 11: "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-none-manylinux_x86_64.whl", - "pkg-0.0.1-py3-none-manylinux_x86_64.whl", - "pkg-0.0.1-py311-none-manylinux_x86_64.whl", - "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", - ], - want_platforms = ["cp3{}_linux_x86_64".format(minor_version)], + got = _select_whl( + whls = whls, + platforms = ["manylinux_x86_64"], + want_abis = ["none"], + python_version = "3.{}".format(minor_version), ) - _match(env, got, match) + _match(env, [got], match) _tests.append(_test_supported_cp_version_manylinux) def _test_ignore_unsupported(env): - got = _select_whls( - whls = [ - "pkg-0.0.1-xx3-abi3-any.whl", - ], - want_platforms = ["cp30_ignored"], + whls = ["pkg-0.0.1-xx3-abi3-any.whl"] + got = _select_whl( + whls = whls, + platforms = ["any"], + want_abis = ["none"], + python_version = "3.0", ) - _match(env, got) + if got: + _match(env, [got], None) _tests.append(_test_ignore_unsupported) def _test_match_abi_and_not_py_version(env): # Check we match the ABI and not the py version - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp37_linux_x86_64"]) + whls = WHL_LIST + platforms = [ + "musllinux_*_x86_64", + "manylinux_*_x86_64", + ] + got = _select_whl( + whls = whls, + platforms = platforms, + want_abis = ["abi3", "cp37m"], + python_version = "3.7", + ) _match( env, - got, + [got], "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", ) -_tests.append(_test_match_abi_and_not_py_version) - -def _test_select_filename_with_many_tags(env): - # Check we can select a filename with many platform tags - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp39_linux_x86_32"]) + got = _select_whl( + whls = whls, + platforms = platforms[::-1], + want_abis = ["abi3", "cp37m"], + python_version = "3.7", + ) _match( env, - got, - "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", + [got], + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", ) -_tests.append(_test_select_filename_with_many_tags) +_tests.append(_test_match_abi_and_not_py_version) -def _test_osx_prefer_arch_specific(env): - # Check that we prefer the specific wheel - got = _select_whls( +def _test_select_filename_with_many_tags(env): + # Check we can select a filename with many platform tags + got = _select_whl( whls = WHL_LIST, - want_platforms = ["cp311_osx_x86_64", "cp311_osx_x86_32"], + platforms = [ + "any", + "musllinux_*_i686", + "manylinux_*_i686", + ], + want_abis = ["none", "abi3", "cp39"], + python_version = "3.9", + limit = 5, ) _match( env, got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", "pkg-0.0.1-py3-none-any.whl", - ) - - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp311_osx_aarch64"]) - _match( - env, - got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", + "pkg-0.0.1-py3-abi3-any.whl", "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", ) -_tests.append(_test_osx_prefer_arch_specific) +_tests.append(_test_select_filename_with_many_tags) -def _test_osx_fallback_to_universal2(env): - # Check that we can use the universal2 if the arm wheel is not available - got = _select_whls( - whls = [w for w in WHL_LIST if "arm64" not in w], - want_platforms = ["cp311_osx_aarch64"], +def _test_freethreaded_wheels(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = WHL_LIST, + platforms = [ + "any", + "musllinux_*_x86_64", + ], + want_abis = ["none", "abi3", "cp313", "cp313t"], + python_version = "3.13", + limit = 8, ) _match( env, got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp39-abi3-any.whl", + # The last item has the most priority "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_osx_fallback_to_universal2) - -def _test_prefer_manylinux_wheels(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp39_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py310-abi3-any.whl", "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_prefer_manylinux_wheels) - -def _test_freethreaded_wheels(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_freethreaded_wheels) - -def _test_micro_version_freethreaded(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313.3_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", ) -_tests.append(_test_micro_version_freethreaded) +_tests.append(_test_freethreaded_wheels) def select_whl_test_suite(name): """Create the test suite. diff --git a/tests/pypi/whl_target_platforms/BUILD.bazel b/tests/pypi/whl_target_platforms/BUILD.bazel index 6c35b08d32..fec25af033 100644 --- a/tests/pypi/whl_target_platforms/BUILD.bazel +++ b/tests/pypi/whl_target_platforms/BUILD.bazel @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":select_whl_tests.bzl", "select_whl_test_suite") load(":whl_target_platforms_tests.bzl", "whl_target_platforms_test_suite") -select_whl_test_suite(name = "select_whl_tests") - whl_target_platforms_test_suite(name = "whl_target_platforms_tests") From 314159014aa0611295088724083d9a27d3e520a0 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 00:43:40 +0900 Subject: [PATCH 04/29] chore(pypi): remove unused config setting code Stacked on #3058 Since the parent PR is simplifying the code a great deal, we can remove the unused code and its tests. Hopefully this makes the whole codebase easier to contribute to. There is probably more cleanup that can be done, but since the CI is now green, it's a good start. --- python/private/pypi/BUILD.bazel | 6 - python/private/pypi/config_settings.bzl | 282 +-------- python/private/pypi/flags.bzl | 50 +- python/private/pypi/hub_repository.bzl | 2 - python/private/pypi/pkg_aliases.bzl | 139 +---- python/private/pypi/render_pkg_aliases.bzl | 50 +- python/private/pypi/whl_config_setting.bzl | 4 +- .../config_settings/config_settings_tests.bzl | 560 +----------------- tests/pypi/extension/extension_tests.bzl | 2 - tests/pypi/pkg_aliases/pkg_aliases_test.bzl | 192 ------ .../render_pkg_aliases_test.bzl | 270 --------- 11 files changed, 18 insertions(+), 1539 deletions(-) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 982c64436e..66659c7d78 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -62,8 +62,6 @@ bzl_library( name = "config_settings_bzl", srcs = ["config_settings.bzl"], deps = [ - ":flags_bzl", - "//python/private:flags_bzl", "@bazel_skylib//lib:selects", ], ) @@ -321,8 +319,6 @@ bzl_library( srcs = ["pkg_aliases.bzl"], deps = [ ":labels_bzl", - ":parse_whl_name_bzl", - ":whl_target_platforms_bzl", "//python/private:text_util_bzl", "@bazel_skylib//lib:selects", ], @@ -342,9 +338,7 @@ bzl_library( srcs = ["render_pkg_aliases.bzl"], deps = [ ":generate_group_library_build_bazel_bzl", - ":parse_whl_name_bzl", ":whl_config_setting_bzl", - ":whl_target_platforms_bzl", "//python/private:normalize_name_bzl", "//python/private:text_util_bzl", ], diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl index f4826007f8..47de287071 100644 --- a/python/private/pypi/config_settings.bzl +++ b/python/private/pypi/config_settings.bzl @@ -16,102 +16,13 @@ The {obj}`config_settings` macro is used to create the config setting targets that can be used in the {obj}`pkg_aliases` macro for selecting the compatible repositories. - -Bazel's selects work by selecting the most-specialized configuration setting -that matches the target platform, which is further described in [bazel documentation][docs]. -We can leverage this fact to ensure that the most specialized matches are used -by default with the users being able to configure string_flag values to select -the less specialized ones. - -[docs]: https://bazel.build/docs/configurable-attributes - -The config settings in the order from the least specialized to the most -specialized is as follows: -* `:is_cp3` -* `:is_cp3_sdist` -* `:is_cp3_py_none_any` -* `:is_cp3_py3_none_any` -* `:is_cp3_py3_abi3_any` -* `:is_cp3_none_any` -* `:is_cp3_any_any` -* `:is_cp3_cp3_any` and `:is_cp3_cp3t_any` -* `:is_cp3_py_none_` -* `:is_cp3_py3_none_` -* `:is_cp3_py3_abi3_` -* `:is_cp3_none_` -* `:is_cp3_abi3_` -* `:is_cp3_cp3_` and `:is_cp3_cp3t_` - -Optionally instead of `` there sometimes may be `.` used in order to fully specify the versions - -The specialization of free-threaded vs non-free-threaded wheels is the same as -they are just variants of each other. The same goes for the specialization of -`musllinux` vs `manylinux`. - -The goal of this macro is to provide config settings that provide unambigous -matches if any pair of them is used together for any target configuration -setting. We achieve this by using dummy internal `flag_values` keys to force the -items further down the list to appear to be more specialized than the ones above. - -What is more, the names of the config settings are as similar to the platform wheel -specification as possible. How the wheel names map to the config setting names defined -in here is described in {obj}`pkg_aliases` documentation. - -:::{note} -Right now the specialization of adjacent config settings where one is with -`constraint_values` and one is without is ambiguous. I.e. `py_none_any` and -`sdist_linux_x86_64` have the same specialization from bazel point of view -because one has one `flag_value` entry and `constraint_values` and the -other has 2 flag_value entries. And unfortunately there is no way to disambiguate -it, because we are essentially in two dimensions here (`flag_values` and -`constraint_values`). Hence, when using the `config_settings` from here, -either have all of them with empty `suffix` or all of them with a non-empty -suffix. -::: """ load("@bazel_skylib//lib:selects.bzl", "selects") -load("//python/private:flags.bzl", "LibcFlag") -load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag") - -FLAGS = struct( - **{ - f: str(Label("//python/config_settings:" + f)) - for f in [ - "is_pip_whl_auto", - "is_pip_whl_no", - "is_pip_whl_only", - "_is_py_freethreaded_yes", - "_is_py_freethreaded_no", - "pip_whl_glibc_version", - "pip_whl_muslc_version", - "pip_whl_osx_arch", - "pip_whl_osx_version", - "py_linux_libc", - "python_version", - ] - } -) - -_DEFAULT = "//conditions:default" -_INCOMPATIBLE = "@platforms//:incompatible" - -# Here we create extra string flags that are just to work with the select -# selecting the most specialized match. We don't allow the user to change -# them. -_flags = struct( - **{ - f: str(Label("//python/config_settings:_internal_pip_" + f)) - for f in INTERNAL_FLAGS - } -) def config_settings( *, python_versions = [], - glibc_versions = [], - muslc_versions = [], - osx_versions = [], name = None, platform_config_settings = {}, **kwargs): @@ -121,12 +32,6 @@ def config_settings( name (str): Currently unused. python_versions (list[str]): The list of python versions to configure config settings for. - glibc_versions (list[str]): The list of glibc version of the wheels to - configure config settings for. - muslc_versions (list[str]): The list of musl version of the wheels to - configure config settings for. - osx_versions (list[str]): The list of OSX OS versions to configure - config settings for. platform_config_settings: {type}`dict[str, list[str]]` the constraint values to use instead of the default ones. Key are platform names (a human-friendly platform string). Values are lists of @@ -135,48 +40,22 @@ def config_settings( {obj}`native`. """ - glibc_versions = [""] + glibc_versions - muslc_versions = [""] + muslc_versions - osx_versions = [""] + osx_versions target_platforms = { "": [], - # TODO @aignas 2025-06-15: allowing universal2 and platform specific wheels in one - # closure is making things maybe a little bit too complicated. - "osx_universal2": ["@platforms//os:osx"], } | platform_config_settings for python_version in python_versions: for platform_name, config_settings in target_platforms.items(): suffix = "_{}".format(platform_name) if platform_name else "" - os, _, cpu = platform_name.partition("_") - - # We parse the target settings and if there is a "platforms//os" or - # "platforms//cpu" value in here, we also add it into the constraint_values - # - # this is to ensure that we can still pass all of the unit tests for config - # setting specialization. - constraint_values = [] - for setting in config_settings: - setting_label = Label(setting) - if setting_label.repo_name == "platforms" and setting_label.package in ["os", "cpu"]: - constraint_values.append(setting) _dist_config_settings( suffix = suffix, - plat_flag_values = _plat_flag_values( - os = os, - cpu = cpu, - osx_versions = osx_versions, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - ), config_settings = config_settings, - constraint_values = constraint_values, python_version = python_version, **kwargs ) -def _dist_config_settings(*, suffix, plat_flag_values, python_version, **kwargs): +def _dist_config_settings(*, suffix, python_version, **kwargs): flag_values = { Label("//python/config_settings:python_version_major_minor"): python_version, } @@ -190,156 +69,11 @@ def _dist_config_settings(*, suffix, plat_flag_values, python_version, **kwargs) **kwargs ) - flag_values[_flags.dist] = "" - - # First create an sdist, we will be building upon the flag values, which - # will ensure that each sdist config setting is the least specialized of - # all. However, we need at least one flag value to cover the case where we - # have `sdist` for any platform, hence we have a non-empty `flag_values` - # here. - _dist_config_setting( - name = "{}_sdist{}".format(prefix, suffix), - flag_values = flag_values, - compatible_with = (FLAGS.is_pip_whl_no, FLAGS.is_pip_whl_auto), - **kwargs - ) - - used_flags = {} - - # NOTE @aignas 2024-12-01: the abi3 is not compatible with freethreaded - # builds as per PEP703 (https://peps.python.org/pep-0703/#backwards-compatibility) - # - # The discussion here also reinforces this notion: - # https://discuss.python.org/t/pep-703-making-the-global-interpreter-lock-optional-3-12-updates/26503/99 - - for name, f, compatible_with in [ - ("py_none", _flags.whl, None), - ("py3_none", _flags.whl_py3, None), - ("py3_abi3", _flags.whl_py3_abi3, (FLAGS._is_py_freethreaded_no,)), - ("none", _flags.whl_pycp3x, None), - ("abi3", _flags.whl_pycp3x_abi3, (FLAGS._is_py_freethreaded_no,)), - # The below are not specializations of one another, they are variants - (cpv, _flags.whl_pycp3x_abicp, (FLAGS._is_py_freethreaded_no,)), - (cpv + "t", _flags.whl_pycp3x_abicp, (FLAGS._is_py_freethreaded_yes,)), - ]: - if (f, compatible_with) in used_flags: - # This should never happen as all of the different whls should have - # unique flag values - fail("BUG: the flag {} is attempted to be added twice to the list".format(f)) - else: - flag_values[f] = "yes" if f == _flags.whl else "" - used_flags[(f, compatible_with)] = True - - _dist_config_setting( - name = "{}_{}_any{}".format(prefix, name, suffix), - flag_values = flag_values, - compatible_with = compatible_with, - **kwargs - ) - - generic_flag_values = flag_values - generic_used_flags = used_flags - - for (suffix, flag_values) in plat_flag_values: - used_flags = {(f, None): True for f in flag_values} | generic_used_flags - flag_values = flag_values | generic_flag_values - - for name, f, compatible_with in [ - ("py_none", _flags.whl_plat, None), - ("py3_none", _flags.whl_plat_py3, None), - ("py3_abi3", _flags.whl_plat_py3_abi3, (FLAGS._is_py_freethreaded_no,)), - ("none", _flags.whl_plat_pycp3x, None), - ("abi3", _flags.whl_plat_pycp3x_abi3, (FLAGS._is_py_freethreaded_no,)), - # The below are not specializations of one another, they are variants - (cpv, _flags.whl_plat_pycp3x_abicp, (FLAGS._is_py_freethreaded_no,)), - (cpv + "t", _flags.whl_plat_pycp3x_abicp, (FLAGS._is_py_freethreaded_yes,)), - ]: - if (f, compatible_with) in used_flags: - # This should never happen as all of the different whls should have - # unique flag values. - fail("BUG: the flag {} is attempted to be added twice to the list".format(f)) - else: - flag_values[f] = "" - used_flags[(f, compatible_with)] = True - - _dist_config_setting( - name = "{}_{}_{}".format(prefix, name, suffix), - flag_values = flag_values, - compatible_with = compatible_with, - **kwargs - ) - -def _to_version_string(version, sep = "."): - if not version: - return "" - - return "{}{}{}".format(version[0], sep, version[1]) - -def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions): - ret = [] - if os == "": - return [] - elif os == "windows": - ret.append(("{}_{}".format(os, cpu), {})) - elif os == "osx": - for osx_version in osx_versions: - flags = { - FLAGS.pip_whl_osx_version: _to_version_string(osx_version), - } - if cpu != "universal2": - flags[FLAGS.pip_whl_osx_arch] = UniversalWhlFlag.ARCH - - if not osx_version: - suffix = "{}_{}".format(os, cpu) - else: - suffix = "{}_{}_{}".format(os, _to_version_string(osx_version, "_"), cpu) - - ret.append((suffix, flags)) - - elif os == "linux": - for os_prefix, linux_libc in { - os: LibcFlag.GLIBC, - "many" + os: LibcFlag.GLIBC, - "musl" + os: LibcFlag.MUSL, - }.items(): - if linux_libc == LibcFlag.GLIBC: - libc_versions = glibc_versions - libc_flag = FLAGS.pip_whl_glibc_version - elif linux_libc == LibcFlag.MUSL: - libc_versions = muslc_versions - libc_flag = FLAGS.pip_whl_muslc_version - else: - fail("Unsupported libc type: {}".format(linux_libc)) - - for libc_version in libc_versions: - if libc_version and os_prefix == os: - continue - elif libc_version: - suffix = "{}_{}_{}".format(os_prefix, _to_version_string(libc_version, "_"), cpu) - else: - suffix = "{}_{}".format(os_prefix, cpu) - - ret.append(( - suffix, - { - FLAGS.py_linux_libc: linux_libc, - libc_flag: _to_version_string(libc_version), - }, - )) - else: - fail("Unsupported os: {}".format(os)) - - return ret - -def _dist_config_setting(*, name, compatible_with = None, selects = selects, native = native, config_settings = None, **kwargs): +def _dist_config_setting(*, name, selects = selects, native = native, config_settings = None, **kwargs): """A macro to create a target for matching Python binary and source distributions. Args: name: The name of the public target. - compatible_with: {type}`tuple[Label]` A collection of config settings that are - compatible with the given dist config setting. For example, if only - non-freethreaded python builds are allowed, add - FLAGS._is_py_freethreaded_no here. config_settings: {type}`list[str | Label]` the list of target settings that must be matched before we try to evaluate the config_setting that we may create in this function. @@ -352,18 +86,6 @@ def _dist_config_setting(*, name, compatible_with = None, selects = selects, nat **kwargs: The kwargs passed to the config_setting rule. Visibility of the main alias target is also taken from the kwargs. """ - if compatible_with: - dist_config_setting_name = "_" + name - native.alias( - name = name, - actual = select( - {setting: dist_config_setting_name for setting in compatible_with} | { - _DEFAULT: _INCOMPATIBLE, - }, - ), - visibility = kwargs.get("visibility"), - ) - name = dist_config_setting_name # first define the config setting that has all of the constraint values _name = "_" + name diff --git a/python/private/pypi/flags.bzl b/python/private/pypi/flags.bzl index 037383910e..8aa5c94dd3 100644 --- a/python/private/pypi/flags.bzl +++ b/python/private/pypi/flags.bzl @@ -18,7 +18,7 @@ NOTE: The transitive loads of this should be kept minimal. This avoids loading unnecessary files when all that are needed are flag definitions. """ -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo", "string_flag") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("//python/private:enum.bzl", "enum") load(":env_marker_info.bzl", "EnvMarkerInfo") load( @@ -53,64 +53,16 @@ UniversalWhlFlag = enum( UNIVERSAL = "universal", ) -_STRING_FLAGS = [ - "dist", - "whl_plat", - "whl_plat_py3", - "whl_plat_py3_abi3", - "whl_plat_pycp3x", - "whl_plat_pycp3x_abi3", - "whl_plat_pycp3x_abicp", - "whl_py3", - "whl_py3_abi3", - "whl_pycp3x", - "whl_pycp3x_abi3", - "whl_pycp3x_abicp", -] - -INTERNAL_FLAGS = [ - "whl", -] + _STRING_FLAGS - def define_pypi_internal_flags(name): """define internal PyPI flags used in PyPI hub repository by pkg_aliases. Args: name: not used """ - for flag in _STRING_FLAGS: - string_flag( - name = "_internal_pip_" + flag, - build_setting_default = "", - values = [""], - visibility = ["//visibility:public"], - ) - - _allow_wheels_flag( - name = "_internal_pip_whl", - visibility = ["//visibility:public"], - ) - _default_env_marker_config( name = "_pip_env_marker_default_config", ) -def _allow_wheels_flag_impl(ctx): - input = ctx.attr._setting[BuildSettingInfo].value - value = "yes" if input in ["auto", "only"] else "no" - return [config_common.FeatureFlagInfo(value = value)] - -_allow_wheels_flag = rule( - implementation = _allow_wheels_flag_impl, - attrs = { - "_setting": attr.label(default = "//python/config_settings:pip_whl"), - }, - doc = """\ -This rule allows us to greatly reduce the number of config setting targets at no cost even -if we are duplicating some of the functionality of the `native.config_setting`. -""", -) - def _default_env_marker_config(**kwargs): _env_marker_config( os_name = select(os_name_select_map), diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 75f3ec98d7..5b46aeeb61 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -144,8 +144,6 @@ def _whl_config_setting_dict(a): ret = {} if a.config_setting: ret["config_setting"] = a.config_setting - if a.filename: - ret["filename"] = a.filename if a.target_platforms: ret["target_platforms"] = a.target_platforms if a.version: diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index d71c37cb4b..23e1fc67c8 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -25,8 +25,6 @@ Definitions: :cpu: CPU architecture identifier that exists in `@platforms//cpu:`. :python_tag: The Python tag as defined by the [Python Packaging Authority][packaging_spec]. E.g. `py2.py3`, `py3`, `py311`, `cp311`. :abi_tag: The ABI tag as defined by the [Python Packaging Authority][packaging_spec]. E.g. `none`, `abi3`, `cp311`, `cp311t`. -:platform_tag: The Platform tag as defined by the [Python Packaging Authority][packaging_spec]. E.g. `manylinux_2_17_x86_64`. -:platform_suffix: is a derivative of the `platform_tag` and is used to implement selection based on `libc` or `osx` version. All of the config settings used by this macro are generated by {obj}`config_settings`, for more detailed documentation on what each config @@ -84,14 +82,6 @@ load( "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) -load(":parse_whl_name.bzl", "parse_whl_name") -load(":whl_target_platforms.bzl", "whl_target_platforms") - -# This value is used as sentinel value in the alias/config setting machinery -# for libc and osx versions. If we encounter this version in this part of the -# code, then it means that we have a bug in rules_python and that we should fix -# it. It is more of an internal consistency check. -_VERSION_NONE = (0, 0) _NO_MATCH_ERROR_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -217,21 +207,9 @@ def pkg_aliases( actual = "//_groups:{}_whl".format(group_name), ) -def _normalize_versions(name, versions): - if not versions: - return [] - - if _VERSION_NONE in versions: - fail("a sentinel version found in '{}', check render_pkg_aliases for bugs".format(name)) - - return sorted(versions) - def multiplatform_whl_aliases( *, - aliases = [], - glibc_versions = [], - muslc_versions = [], - osx_versions = []): + aliases = []): """convert a list of aliases from filename to config_setting ones. Exposed only for unit tests. @@ -241,12 +219,6 @@ def multiplatform_whl_aliases( to process. Any aliases that have the filename set will be converted to a dict of config settings to repo names. The struct is created by {func}`whl_config_setting`. - glibc_versions: {type}`list[tuple[int, int]]` list of versions that can be - used in this hub repo. - muslc_versions: {type}`list[tuple[int, int]]` list of versions that can be - used in this hub repo. - osx_versions: {type}`list[tuple[int, int]]` list of versions that can be - used in this hub repo. Returns: A dict with of config setting labels to repo names or the repo name itself. @@ -256,33 +228,19 @@ def multiplatform_whl_aliases( # We don't have any aliases, this is a repo name return aliases - # TODO @aignas 2024-11-17: we might be able to use FeatureFlagInfo and some - # code gen to create a version_lt_x target, which would allow us to check - # if the libc version is in a particular range. - glibc_versions = _normalize_versions("glibc_versions", glibc_versions) - muslc_versions = _normalize_versions("muslc_versions", muslc_versions) - osx_versions = _normalize_versions("osx_versions", osx_versions) - ret = {} versioned_additions = {} for alias, repo in aliases.items(): if type(alias) != "struct": ret[alias] = repo continue - elif not (alias.filename or alias.target_platforms): + elif not alias.target_platforms: # This is an internal consistency check - fail("Expected to have either 'filename' or 'target_platforms' set, got: {}".format(alias)) + fail("Expected to have 'target_platforms' set, got: {}".format(alias)) config_settings, all_versioned_settings = get_filename_config_settings( - filename = alias.filename or "", target_platforms = alias.target_platforms, python_version = alias.version, - # If we have multiple platforms but no wheel filename, lets use different - # config settings. - non_whl_prefix = "sdist" if alias.filename else "", - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, ) for setting in config_settings: @@ -329,70 +287,28 @@ def multiplatform_whl_aliases( def get_filename_config_settings( *, - filename, target_platforms, - python_version, - glibc_versions = None, - muslc_versions = None, - osx_versions = None, - non_whl_prefix = "sdist"): + python_version): """Get the filename config settings. Exposed only for unit tests. Args: - filename: the distribution filename (can be a whl or an sdist). target_platforms: list[str], target platforms in "{abi}_{os}_{cpu}" format. - glibc_versions: list[tuple[int, int]], list of versions. - muslc_versions: list[tuple[int, int]], list of versions. - osx_versions: list[tuple[int, int]], list of versions. python_version: the python version to generate the config_settings for. - non_whl_prefix: the prefix of the config setting when the whl we don't have - a filename ending with ".whl". Returns: A tuple: * A list of config settings that are generated by ./pip_config_settings.bzl * The list of default version settings. """ - prefixes = [] suffixes = [] setting_supported_versions = {} - - if filename.endswith(".whl"): - parsed = parse_whl_name(filename) - if parsed.python_tag == "py2.py3": - py = "py_" - elif parsed.python_tag == "py3": - py = "py3_" - elif parsed.python_tag.startswith("cp"): - py = "" - else: - py = "py3_" - - abi = parsed.abi_tag - - # TODO @aignas 2025-04-20: test - abi, _, _ = abi.partition(".") - - if parsed.platform_tag == "any": - prefixes = ["{}{}_any".format(py, abi)] - else: - prefixes = ["{}{}".format(py, abi)] - suffixes = _whl_config_setting_suffixes( - platform_tag = parsed.platform_tag, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, - setting_supported_versions = setting_supported_versions, - ) - else: - prefixes = [non_whl_prefix or ""] - + prefixes = [""] py = "cp{}".format(python_version).replace(".", "") prefixes = [ "{}_{}".format(py, prefix) if prefix else py - for prefix in prefixes + for prefix in [""] ] versioned = { @@ -415,49 +331,6 @@ def get_filename_config_settings( else: return [":is_{}".format(p) for p in prefixes], setting_supported_versions -def _whl_config_setting_suffixes( - platform_tag, - glibc_versions, - muslc_versions, - osx_versions, - setting_supported_versions): - suffixes = [] - for platform_tag in platform_tag.split("."): - for p in whl_target_platforms(platform_tag): - prefix = p.os - suffix = p.cpu - if "manylinux" in platform_tag: - prefix = "manylinux" - versions = glibc_versions - elif "musllinux" in platform_tag: - prefix = "musllinux" - versions = muslc_versions - elif p.os in ["linux", "windows"]: - versions = [(0, 0)] - elif p.os == "osx": - versions = osx_versions - if "universal2" in platform_tag: - suffix = "universal2" - else: - fail("Unsupported whl os: {}".format(p.os)) - - default_version_setting = "{}_{}".format(prefix, suffix) - supported_versions = {} - for v in versions: - if v == (0, 0): - suffixes.append(default_version_setting) - elif v >= p.version: - supported_versions[v] = "{}_{}_{}_{}".format( - prefix, - v[0], - v[1], - suffix, - ) - if supported_versions: - setting_supported_versions[default_version_setting] = supported_versions - - return suffixes - def _non_versioned_platform(p, *, strict = False): """A small utility function that converts 'cp311_linux_x86_64' to 'linux_x86_64'. diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index e743fc20f7..77063921c8 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -22,8 +22,6 @@ load( ":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel", ) # buildifier: disable=bzl-visibility -load(":parse_whl_name.bzl", "parse_whl_name") -load(":whl_target_platforms.bzl", "whl_target_platforms") NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -48,11 +46,10 @@ def _repr_dict(*, value_repr = repr, **kwargs): return {k: value_repr(v) for k, v in kwargs.items() if v} def _repr_config_setting(alias): - if alias.filename or alias.target_platforms: + if alias.target_platforms: return render.call( "whl_config_setting", **_repr_dict( - filename = alias.filename, target_platforms = alias.target_platforms, config_setting = alias.config_setting, version = alias.version, @@ -179,15 +176,9 @@ def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, contents = render_pkg_aliases( aliases = aliases, - glibc_versions = flag_versions.get("glibc_versions", []), - muslc_versions = flag_versions.get("muslc_versions", []), - osx_versions = flag_versions.get("osx_versions", []), **kwargs ) contents["_config/BUILD.bazel"] = _render_config_settings( - glibc_versions = flag_versions.get("glibc_versions", []), - muslc_versions = flag_versions.get("muslc_versions", []), - osx_versions = flag_versions.get("osx_versions", []), python_versions = _major_minor_versions(flag_versions.get("python_versions", [])), platform_config_settings = platform_config_settings, visibility = ["//:__subpackages__"], @@ -219,54 +210,21 @@ def get_whl_flag_versions(settings): * python_versions """ python_versions = {} - glibc_versions = {} target_platforms = {} - muslc_versions = {} - osx_versions = {} for setting in settings: - if not setting.version and not setting.filename: + if not setting.version: continue if setting.version: python_versions[setting.version] = None - if setting.filename and setting.filename.endswith(".whl") and not setting.filename.endswith("-any.whl"): - parsed = parse_whl_name(setting.filename) - else: - for plat in setting.target_platforms or []: - target_platforms[_non_versioned_platform(plat)] = None - continue - - for platform_tag in parsed.platform_tag.split("."): - parsed = whl_target_platforms(platform_tag) - - for p in parsed: - target_platforms[p.target_platform] = None - - if platform_tag.startswith("win") or platform_tag.startswith("linux"): - continue - - head, _, tail = platform_tag.partition("_") - major, _, tail = tail.partition("_") - minor, _, tail = tail.partition("_") - if tail: - version = (int(major), int(minor)) - if "many" in head: - glibc_versions[version] = None - elif "musl" in head: - muslc_versions[version] = None - elif "mac" in head: - osx_versions[version] = None - else: - fail(platform_tag) + for plat in setting.target_platforms or []: + target_platforms[_non_versioned_platform(plat)] = None return { k: sorted(v) for k, v in { - "glibc_versions": glibc_versions, - "muslc_versions": muslc_versions, - "osx_versions": osx_versions, "python_versions": python_versions, "target_platforms": target_platforms, }.items() diff --git a/python/private/pypi/whl_config_setting.bzl b/python/private/pypi/whl_config_setting.bzl index 3b81e4694f..da6dd80dc1 100644 --- a/python/private/pypi/whl_config_setting.bzl +++ b/python/private/pypi/whl_config_setting.bzl @@ -14,7 +14,7 @@ "A small function to create an alias for a whl distribution" -def whl_config_setting(*, version = None, config_setting = None, filename = None, target_platforms = None): +def whl_config_setting(*, version = None, config_setting = None, target_platforms = None): """The bzl_packages value used by by the render_pkg_aliases function. This contains the minimum amount of information required to generate correct @@ -27,7 +27,6 @@ def whl_config_setting(*, version = None, config_setting = None, filename = None is no match found during a select. config_setting: {type}`str | Label | None` the config setting that we should use. Defaults to "//_config:is_python_{version}". - filename: {type}`str | None` the distribution filename to derive the config_setting. target_platforms: {type}`list[str] | None` the list of target_platforms for this distribution. @@ -51,7 +50,6 @@ def whl_config_setting(*, version = None, config_setting = None, filename = None return struct( config_setting = config_setting, - filename = filename, # Make the struct hashable target_platforms = tuple(target_platforms) if target_platforms else None, version = version, diff --git a/tests/pypi/config_settings/config_settings_tests.bzl b/tests/pypi/config_settings/config_settings_tests.bzl index a15f6b4d32..3f2dbd26e7 100644 --- a/tests/pypi/config_settings/config_settings_tests.bzl +++ b/tests/pypi/config_settings/config_settings_tests.bzl @@ -78,9 +78,9 @@ def _test_legacy_default(name): _analysis_test( name = name, dist = { - "is_cp37": "legacy", + "is_cp37": "foo", }, - want = "legacy", + want = "foo", ) _tests.append(_test_legacy_default) @@ -89,562 +89,13 @@ def _test_legacy_with_constraint_values(name): _analysis_test( name = name, dist = { - "is_cp37": "legacy", - "is_cp37_linux_aarch64": "legacy_platform_override", + "is_cp37_linux_aarch64": "foo", }, - want = "legacy_platform_override", + want = "foo", ) _tests.append(_test_legacy_with_constraint_values) -# Tests when we only have an `sdist` present. - -def _test_sdist_default(name): - _analysis_test( - name = name, - dist = { - "is_cp37_sdist": "sdist", - }, - want = "sdist", - ) - -_tests.append(_test_sdist_default) - -def _test_legacy_less_specialized_than_sdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37": "legacy", - "is_cp37_sdist": "sdist", - }, - want = "sdist", - ) - -_tests.append(_test_legacy_less_specialized_than_sdist) - -def _test_sdist_no_whl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("no"), - ], - want = "sdist", - ) - -_tests.append(_test_sdist_no_whl) - -def _test_sdist_no_sdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("only"), - ], - # We will use `no_match_error` in the real case to indicate that `sdist` is not - # allowed to be used. - want = "no_match", - ) - -_tests.append(_test_sdist_no_sdist) - -def _test_basic_whl_default(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py_none_any": "whl", - "is_cp37_sdist": "sdist", - }, - want = "whl", - ) - -_tests.append(_test_basic_whl_default) - -def _test_basic_whl_nowhl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py_none_any": "whl", - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("no"), - ], - want = "sdist", - ) - -_tests.append(_test_basic_whl_nowhl) - -def _test_basic_whl_nosdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py_none_any": "whl", - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("only"), - ], - want = "whl", - ) - -_tests.append(_test_basic_whl_nosdist) - -def _test_whl_default(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "whl", - "is_cp37_py_none_any": "basic_whl", - }, - want = "whl", - ) - -_tests.append(_test_whl_default) - -def _test_whl_nowhl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "whl", - "is_cp37_py_none_any": "basic_whl", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("no"), - ], - want = "no_match", - ) - -_tests.append(_test_whl_nowhl) - -def _test_whl_nosdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "whl", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("only"), - ], - want = "whl", - ) - -_tests.append(_test_whl_nosdist) - -def _test_abi_whl_is_prefered(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_abi3_any": "abi_whl", - "is_cp37_py3_none_any": "whl", - }, - want = "abi_whl", - ) - -_tests.append(_test_abi_whl_is_prefered) - -def _test_whl_with_constraints_is_prefered(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "default_whl", - "is_cp37_py3_none_any_linux_aarch64": "whl", - "is_cp37_py3_none_any_linux_x86_64": "amd64_whl", - }, - want = "whl", - ) - -_tests.append(_test_whl_with_constraints_is_prefered) - -def _test_cp_whl_is_prefered_over_py3(name): - _analysis_test( - name = name, - dist = { - "is_cp37_none_any": "cp", - "is_cp37_py3_abi3_any": "py3_abi3", - "is_cp37_py3_none_any": "py3", - }, - want = "cp", - ) - -_tests.append(_test_cp_whl_is_prefered_over_py3) - -def _test_cp_abi_whl_is_prefered_over_py3(name): - _analysis_test( - name = name, - dist = { - "is_cp37_abi3_any": "cp", - "is_cp37_py3_abi3_any": "py3", - }, - want = "cp", - ) - -_tests.append(_test_cp_abi_whl_is_prefered_over_py3) - -def _test_cp_version_is_selected_when_python_version_is_specified(name): - _analysis_test( - name = name, - dist = { - "is_cp310_none_any": "cp310", - "is_cp38_none_any": "cp38", - "is_cp39_none_any": "cp39", - }, - want = "cp310", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_cp_version_is_selected_when_python_version_is_specified) - -def _test_py_none_any_versioned(name): - _analysis_test( - name = name, - dist = { - "is_cp310_py_none_any": "whl", - "is_cp39_py_none_any": "too-low", - }, - want = "whl", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_py_none_any_versioned) - -def _test_cp_whl_is_not_prefered_over_py3_non_freethreaded(name): - _analysis_test( - name = name, - dist = { - "is_cp37_abi3_any": "py3_abi3", - "is_cp37_cp37t_any": "cp", - "is_cp37_none_any": "py3", - }, - want = "py3_abi3", - config_settings = [ - _flag.py_freethreaded("no"), - ], - ) - -_tests.append(_test_cp_whl_is_not_prefered_over_py3_non_freethreaded) - -def _test_cp_whl_is_not_prefered_over_py3_freethreaded(name): - _analysis_test( - name = name, - dist = { - "is_cp37_abi3_any": "py3_abi3", - "is_cp37_cp37_any": "cp", - "is_cp37_none_any": "py3", - }, - want = "py3", - config_settings = [ - _flag.py_freethreaded("yes"), - ], - ) - -_tests.append(_test_cp_whl_is_not_prefered_over_py3_freethreaded) - -def _test_cp_cp_whl(name): - _analysis_test( - name = name, - dist = { - "is_cp310_cp310_linux_aarch64": "whl", - }, - want = "whl", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_cp_cp_whl) - -def _test_cp_version_sdist_is_selected(name): - _analysis_test( - name = name, - dist = { - "is_cp310_sdist": "sdist", - }, - want = "sdist", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_cp_version_sdist_is_selected) - -# NOTE: Right now there is no way to get the following behaviour without -# breaking other tests. We need to choose either ta have the correct -# specialization behaviour between `is_cp37_cp37_any` and -# `is_cp37_cp37_any_linux_aarch64` or this commented out test case. -# -# I think having this behaviour not working is fine because the `suffix` -# will be either present on all of config settings of the same platform -# or none, because we use it as a way to select a separate version of the -# wheel for a single platform only. -# -# If we can think of a better way to handle it, then we can lift this -# limitation. -# -# def _test_any_whl_with_suffix_specialization(name): -# _analysis_test( -# name = name, -# dist = { -# "is_cp37_abi3_any_linux_aarch64": "abi3", -# "is_cp37_cp37_any": "cp37", -# }, -# want = "cp37", -# ) -# -# _tests.append(_test_any_whl_with_suffix_specialization) - -def _test_platform_vs_any_with_suffix_specialization(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_any_linux_aarch64": "any", - "is_cp37_py3_none_linux_aarch64": "platform_whl", - }, - want = "platform_whl", - ) - -_tests.append(_test_platform_vs_any_with_suffix_specialization) - -def _test_platform_whl_is_prefered_over_any_whl_with_constraints(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_abi3_any": "better_default_whl", - "is_cp37_py3_abi3_any_linux_aarch64": "better_default_any_whl", - "is_cp37_py3_none_any": "default_whl", - "is_cp37_py3_none_any_linux_aarch64": "whl", - "is_cp37_py3_none_linux_aarch64": "platform_whl", - }, - want = "platform_whl", - ) - -_tests.append(_test_platform_whl_is_prefered_over_any_whl_with_constraints) - -def _test_abi3_platform_whl_preference(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_abi3_linux_aarch64": "abi3_platform", - "is_cp37_py3_none_linux_aarch64": "platform", - }, - want = "abi3_platform", - ) - -_tests.append(_test_abi3_platform_whl_preference) - -def _test_glibc(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_manylinux_aarch64": "glibc", - "is_cp37_py3_abi3_linux_aarch64": "abi3_platform", - }, - want = "glibc", - ) - -_tests.append(_test_glibc) - -def _test_glibc_versioned(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_manylinux_2_14_aarch64": "glibc", - "is_cp37_cp37_manylinux_2_17_aarch64": "glibc", - "is_cp37_py3_abi3_linux_aarch64": "abi3_platform", - }, - want = "glibc", - config_settings = [ - _flag.py_linux_libc("glibc"), - _flag.pip_whl_glibc_version("2.17"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_glibc_versioned) - -def _test_glibc_compatible_exists(name): - _analysis_test( - name = name, - dist = { - # Code using the conditions will need to construct selects, which - # do the version matching correctly. - "is_cp37_cp37_manylinux_2_14_aarch64": "2_14_whl_via_2_14_branch", - "is_cp37_cp37_manylinux_2_17_aarch64": "2_14_whl_via_2_17_branch", - }, - want = "2_14_whl_via_2_17_branch", - config_settings = [ - _flag.py_linux_libc("glibc"), - _flag.pip_whl_glibc_version("2.17"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_glibc_compatible_exists) - -def _test_musl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_musllinux_aarch64": "musl", - }, - want = "musl", - config_settings = [ - _flag.py_linux_libc("musl"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_musl) - -def _test_windows(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_windows_x86_64": "whl", - "is_cp37_cp37t_windows_x86_64": "whl_freethreaded", - }, - want = "whl", - config_settings = [ - _flag.platform("windows_x86_64"), - ], - ) - -_tests.append(_test_windows) - -def _test_windows_freethreaded(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_windows_x86_64": "whl", - "is_cp37_cp37t_windows_x86_64": "whl_freethreaded", - }, - want = "whl_freethreaded", - config_settings = [ - _flag.platform("windows_x86_64"), - _flag.py_freethreaded("yes"), - ], - ) - -_tests.append(_test_windows_freethreaded) - -def _test_osx(name): - _analysis_test( - name = name, - dist = { - # We prefer arch specific whls over universal - "is_cp37_cp37_osx_universal2": "universal_whl", - "is_cp37_cp37_osx_x86_64": "whl", - }, - want = "whl", - config_settings = [ - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx) - -def _test_osx_universal_default(name): - _analysis_test( - name = name, - dist = { - # We default to universal if only that exists - "is_cp37_cp37_osx_universal2": "whl", - }, - want = "whl", - config_settings = [ - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx_universal_default) - -def _test_osx_universal_only(name): - _analysis_test( - name = name, - dist = { - # If we prefer universal, then we use that - "is_cp37_cp37_osx_universal2": "universal", - "is_cp37_cp37_osx_x86_64": "whl", - }, - want = "universal", - config_settings = [ - _flag.pip_whl_osx_arch("universal"), - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx_universal_only) - -def _test_osx_os_version(name): - _analysis_test( - name = name, - dist = { - # Similarly to the libc version, the user of the config settings will have to - # construct the select so that the version selection is correct. - "is_cp37_cp37_osx_10_9_x86_64": "whl", - }, - want = "whl", - config_settings = [ - _flag.pip_whl_osx_version("10.9"), - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx_os_version) - -def _test_all(name): - _analysis_test( - name = name, - dist = { - "is_cp37_" + f: f - for f in [ - "{py}{abi}_{plat}".format(py = valid_py, abi = valid_abi, plat = valid_plat) - # we have py2.py3, py3, cp3 - for valid_py in ["py_", "py3_", ""] - # cp abi usually comes with a version and we only need one - # config setting variant for all of them because the python - # version will discriminate between different versions. - for valid_abi in ["none", "abi3", "cp37"] - for valid_plat in [ - "any", - "manylinux_2_17_x86_64", - "manylinux_2_17_aarch64", - "osx_x86_64", - "windows_x86_64", - ] - if not ( - valid_abi == "abi3" and valid_py == "py_" or - valid_abi == "cp37" and valid_py != "" - ) - ] - }, - want = "cp37_manylinux_2_17_x86_64", - config_settings = [ - _flag.pip_whl_glibc_version("2.17"), - _flag.platform("linux_x86_64"), - ], - ) - -_tests.append(_test_all) - def config_settings_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, @@ -654,9 +105,6 @@ def config_settings_test_suite(name): # buildifier: disable=function-docstring config_settings( name = "dummy", python_versions = ["3.7", "3.8", "3.9", "3.10"], - glibc_versions = [(2, 14), (2, 17)], - muslc_versions = [(1, 1)], - osx_versions = [(10, 9), (11, 0)], platform_config_settings = { "linux_aarch64": [ "@platforms//cpu:aarch64", diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index ec6b08eb60..7f72d4b8ad 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -968,7 +968,6 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' "cp315_linux_x86_64", ], config_setting = None, - filename = None, ), ], "pypi_315_optimum_osx_aarch64": [ @@ -978,7 +977,6 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' "cp315_osx_aarch64", ], config_setting = None, - filename = None, ), ], }, diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 123ee725f8..0937bc209a 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -157,15 +157,6 @@ def _test_multiplatform_whl_aliases(env): pkg_aliases( name = "bar_baz", actual = { - whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", - version = "3.9", - ): "filename_repo", - whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", - version = "3.9", - target_platforms = ["cp39_linux_x86_64"], - ): "filename_repo_for_platform", whl_config_setting( version = "3.9", target_platforms = ["cp39_linux_x86_64"], @@ -177,9 +168,6 @@ def _test_multiplatform_whl_aliases(env): alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), ), select = mock_select, - glibc_versions = [], - muslc_versions = [], - osx_versions = [], ) # buildifier: disable=unsorted-dict-items @@ -187,8 +175,6 @@ def _test_multiplatform_whl_aliases(env): "pkg": { "//:my_config_setting": "@bzlmod_repo//:pkg", "//_config:is_cp39_linux_x86_64": "@bzlmod_repo_for_a_particular_platform//:pkg", - "//_config:is_cp39_py3_none_any": "@filename_repo//:pkg", - "//_config:is_cp39_py3_none_any_linux_x86_64": "@filename_repo_for_platform//:pkg", "//conditions:default": "_no_matching_repository", }, } @@ -198,8 +184,6 @@ def _test_multiplatform_whl_aliases(env): configuration settings: //:my_config_setting //_config:is_cp39_linux_x86_64 - //_config:is_cp39_py3_none_any - //_config:is_cp39_py3_none_any_linux_x86_64 """) @@ -293,102 +277,6 @@ def _test_multiplatform_whl_aliases_nofilename_target_platforms(env): _tests.append(_test_multiplatform_whl_aliases_nofilename_target_platforms) -def _test_multiplatform_whl_aliases_filename(env): - aliases = { - whl_config_setting( - filename = "foo-0.0.3-py3-none-any.whl", - version = "3.2", - ): "foo-py3-0.0.3", - whl_config_setting( - filename = "foo-0.0.1-py3-none-any.whl", - version = "3.1", - ): "foo-py3-0.0.1", - whl_config_setting( - filename = "foo-0.0.1-cp313-cp313-any.whl", - version = "3.13", - ): "foo-cp-0.0.1", - whl_config_setting( - filename = "foo-0.0.1-cp313-cp313t-any.whl", - version = "3.13", - ): "foo-cpt-0.0.1", - whl_config_setting( - filename = "foo-0.0.2-py3-none-any.whl", - version = "3.1", - target_platforms = [ - "cp31_linux_x86_64", - "cp31_linux_aarch64", - ], - ): "foo-0.0.2", - } - got = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = [], - muslc_versions = [], - osx_versions = [], - ) - want = { - "//_config:is_cp313_cp313_any": "foo-cp-0.0.1", - "//_config:is_cp313_cp313t_any": "foo-cpt-0.0.1", - "//_config:is_cp31_py3_none_any": "foo-py3-0.0.1", - "//_config:is_cp31_py3_none_any_linux_aarch64": "foo-0.0.2", - "//_config:is_cp31_py3_none_any_linux_x86_64": "foo-0.0.2", - "//_config:is_cp32_py3_none_any": "foo-py3-0.0.3", - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_multiplatform_whl_aliases_filename) - -def _test_multiplatform_whl_aliases_filename_versioned(env): - aliases = { - whl_config_setting( - filename = "foo-0.0.1-py3-none-manylinux_2_17_x86_64.whl", - version = "3.1", - ): "glibc-2.17", - whl_config_setting( - filename = "foo-0.0.1-py3-none-manylinux_2_18_x86_64.whl", - version = "3.1", - ): "glibc-2.18", - whl_config_setting( - filename = "foo-0.0.1-py3-none-musllinux_1_1_x86_64.whl", - version = "3.1", - ): "musl-1.1", - } - got = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = [(2, 17), (2, 18)], - muslc_versions = [(1, 1), (1, 2)], - osx_versions = [], - ) - want = { - # This could just work with: - # select({ - # "//_config:is_gt_eq_2.18": "//_config:is_cp3.1_py3_none_manylinux_x86_64", - # "//conditions:default": "//_config:is_gt_eq_2.18", - # }): "glibc-2.18", - # select({ - # "//_config:is_range_2.17_2.18": "//_config:is_cp3.1_py3_none_manylinux_x86_64", - # "//_config:is_glibc_default": "//_config:is_cp3.1_py3_none_manylinux_x86_64", - # "//conditions:default": "//_config:is_glibc_default", - # }): "glibc-2.17", - # ( - # "//_config:is_gt_musl_1.1": "musl-1.1", - # "//_config:is_musl_default": "musl-1.1", - # ): "musl-1.1", - # - # For this to fully work we need to have the pypi:config_settings.bzl to generate the - # extra targets that use the FeatureFlagInfo and this to generate extra aliases for the - # config settings. - "//_config:is_cp31_py3_none_manylinux_2_17_x86_64": "glibc-2.17", - "//_config:is_cp31_py3_none_manylinux_2_18_x86_64": "glibc-2.18", - "//_config:is_cp31_py3_none_manylinux_x86_64": "glibc-2.17", - "//_config:is_cp31_py3_none_musllinux_1_1_x86_64": "musl-1.1", - "//_config:is_cp31_py3_none_musllinux_1_2_x86_64": "musl-1.1", - "//_config:is_cp31_py3_none_musllinux_x86_64": "musl-1.1", - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_multiplatform_whl_aliases_filename_versioned) - def _mock_alias(container): return lambda name, **kwargs: container.append(name) @@ -446,86 +334,6 @@ def _test_config_settings_exist_legacy(env): _tests.append(_test_config_settings_exist_legacy) -def _test_config_settings_exist(env): - for py_tag in ["py2.py3", "py3", "py311", "cp311"]: - if py_tag == "py2.py3": - abis = ["none"] - elif py_tag.startswith("py"): - abis = ["none", "abi3"] - else: - abis = ["none", "abi3", "cp311"] - - for abi_tag in abis: - for platform_tag, kwargs in { - "any": {}, - "macosx_11_0_arm64": { - "osx_versions": [(11, 0)], - "platform_config_settings": { - "osx_aarch64": [ - "@platforms//cpu:aarch64", - "@platforms//os:osx", - ], - }, - }, - "manylinux_2_17_x86_64": { - "glibc_versions": [(2, 17), (2, 18)], - "platform_config_settings": { - "linux_x86_64": [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - }, - }, - "manylinux_2_18_x86_64": { - "glibc_versions": [(2, 17), (2, 18)], - "platform_config_settings": { - "linux_x86_64": [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - }, - }, - "musllinux_1_1_aarch64": { - "muslc_versions": [(1, 2), (1, 1), (1, 0)], - "platform_config_settings": { - "linux_aarch64": [ - "@platforms//cpu:aarch64", - "@platforms//os:linux", - ], - }, - }, - }.items(): - aliases = { - whl_config_setting( - filename = "foo-0.0.1-{}-{}-{}.whl".format(py_tag, abi_tag, platform_tag), - version = "3.11", - ): "repo", - } - available_config_settings = [] - config_settings( - python_versions = ["3.11"], - native = struct( - alias = _mock_alias(available_config_settings), - config_setting = _mock_config_setting([]), - ), - selects = struct( - config_setting_group = _mock_config_setting_group(available_config_settings), - ), - **kwargs - ) - - got_aliases = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = kwargs.get("glibc_versions", []), - muslc_versions = kwargs.get("muslc_versions", []), - osx_versions = kwargs.get("osx_versions", []), - ) - got = [a.partition(":")[-1] for a in got_aliases] - - env.expect.that_collection(available_config_settings).contains_at_least(got) - -_tests.append(_test_config_settings_exist) - def pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index ad7f36aed6..3c6227cfdb 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -15,10 +15,6 @@ """render_pkg_aliases tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load( - "//python/private/pypi:pkg_aliases.bzl", - "get_filename_config_settings", -) # buildifier: disable=bzl-visibility load( "//python/private/pypi:render_pkg_aliases.bzl", "get_whl_flag_versions", @@ -79,17 +75,6 @@ def _test_bzlmod_aliases(env): "cp32_linux_x86_64", ], ): "pypi_32_bar_baz_linux_x86_64", - whl_config_setting( - version = "3.2", - filename = "foo-0.0.0-py3-none-any.whl", - ): "filename_repo", - whl_config_setting( - version = "3.2.2", - filename = "foo-0.0.0-py3-none-any.whl", - target_platforms = [ - "cp32.2_linux_x86_64", - ], - ): "filename_repo_linux_x86_64", }, }, extra_hub_aliases = {"bar_baz": ["foo"]}, @@ -117,15 +102,6 @@ pkg_aliases( config_setting = "//:my_config_setting", version = "3.2", ): "pypi_32_bar_baz_linux_x86_64", - whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", - version = "3.2", - ): "filename_repo", - whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", - target_platforms = ("cp32_linux_x86_64",), - version = "3.2.2", - ): "filename_repo_linux_x86_64", }, extra_aliases = ["foo"], )""" @@ -256,252 +232,6 @@ def _test_get_python_versions_with_target_platforms(env): _tests.append(_test_get_python_versions_with_target_platforms) -def _test_get_python_versions_from_filenames(env): - got = get_whl_flag_versions( - settings = [ - whl_config_setting( - version = "3.3", - filename = "foo-0.0.0-py3-none-" + plat + ".whl", - ) - for plat in [ - "linux_x86_64", - "manylinux_2_17_x86_64", - "manylinux_2_14_aarch64.musllinux_1_1_aarch64", - "musllinux_1_0_x86_64", - "manylinux2014_x86_64.manylinux_2_17_x86_64", - "macosx_11_0_arm64", - "macosx_10_9_x86_64", - "macosx_10_9_universal2", - "windows_x86_64", - ] - ], - ) - want = { - "glibc_versions": [(2, 14), (2, 17)], - "muslc_versions": [(1, 0), (1, 1)], - "osx_versions": [(10, 9), (11, 0)], - "python_versions": ["3.3"], - "target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "osx_x86_64", - "windows_x86_64", - ], - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_get_python_versions_from_filenames) - -def _test_get_flag_versions_from_alias_target_platforms(env): - got = get_whl_flag_versions( - settings = [ - whl_config_setting( - version = "3.3", - filename = "foo-0.0.0-py3-none-" + plat + ".whl", - ) - for plat in [ - "windows_x86_64", - ] - ] + [ - whl_config_setting( - version = "3.3", - filename = "foo-0.0.0-py3-none-any.whl", - target_platforms = [ - "cp33_linux_x86_64", - ], - ), - ], - ) - want = { - "python_versions": ["3.3"], - "target_platforms": [ - "linux_x86_64", - "windows_x86_64", - ], - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_get_flag_versions_from_alias_target_platforms) - -def _test_config_settings( - env, - *, - filename, - want, - python_version, - want_versions = {}, - target_platforms = [], - glibc_versions = [], - muslc_versions = [], - osx_versions = []): - got, got_default_version_settings = get_filename_config_settings( - filename = filename, - target_platforms = target_platforms, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, - python_version = python_version, - ) - env.expect.that_collection(got).contains_exactly(want) - env.expect.that_dict(got_default_version_settings).contains_exactly(want_versions) - -def _test_sdist(env): - # Do the first test for multiple extensions - for ext in [".tar.gz", ".zip"]: - _test_config_settings( - env, - filename = "foo-0.0.1" + ext, - python_version = "3.2", - want = [":is_cp32_sdist"], - ) - - ext = ".zip" - _test_config_settings( - env, - filename = "foo-0.0.1" + ext, - python_version = "3.2", - target_platforms = [ - "linux_aarch64", - "linux_x86_64", - ], - want = [ - ":is_cp32_sdist_linux_aarch64", - ":is_cp32_sdist_linux_x86_64", - ], - ) - -_tests.append(_test_sdist) - -def _test_py2_py3_none_any(env): - _test_config_settings( - env, - filename = "foo-0.0.1-py2.py3-none-any.whl", - python_version = "3.2", - want = [ - ":is_cp32_py_none_any", - ], - ) - - _test_config_settings( - env, - filename = "foo-0.0.1-py2.py3-none-any.whl", - python_version = "3.2", - target_platforms = [ - "osx_x86_64", - ], - want = [":is_cp32_py_none_any_osx_x86_64"], - ) - -_tests.append(_test_py2_py3_none_any) - -def _test_py3_none_any(env): - _test_config_settings( - env, - filename = "foo-0.0.1-py3-none-any.whl", - python_version = "3.1", - want = [":is_cp31_py3_none_any"], - ) - - _test_config_settings( - env, - filename = "foo-0.0.1-py3-none-any.whl", - python_version = "3.1", - target_platforms = ["linux_x86_64"], - want = [":is_cp31_py3_none_any_linux_x86_64"], - ) - -_tests.append(_test_py3_none_any) - -def _test_py3_none_macosx_10_9_universal2(env): - _test_config_settings( - env, - filename = "foo-0.0.1-py3-none-macosx_10_9_universal2.whl", - python_version = "3.1", - osx_versions = [ - (10, 9), - (11, 0), - ], - want = [], - want_versions = { - ":is_cp31_py3_none_osx_universal2": { - (10, 9): ":is_cp31_py3_none_osx_10_9_universal2", - (11, 0): ":is_cp31_py3_none_osx_11_0_universal2", - }, - }, - ) - -_tests.append(_test_py3_none_macosx_10_9_universal2) - -def _test_cp37_abi3_linux_x86_64(env): - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-abi3-linux_x86_64.whl", - python_version = "3.7", - want = [":is_cp37_abi3_linux_x86_64"], - ) - -_tests.append(_test_cp37_abi3_linux_x86_64) - -def _test_cp37_abi3_windows_x86_64(env): - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-abi3-windows_x86_64.whl", - python_version = "3.7", - want = [":is_cp37_abi3_windows_x86_64"], - ) - -_tests.append(_test_cp37_abi3_windows_x86_64) - -def _test_cp37_abi3_manylinux_2_17_x86_64(env): - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", - python_version = "3.7", - glibc_versions = [ - (2, 16), - (2, 17), - (2, 18), - ], - want = [], - want_versions = { - ":is_cp37_abi3_manylinux_x86_64": { - (2, 17): ":is_cp37_abi3_manylinux_2_17_x86_64", - (2, 18): ":is_cp37_abi3_manylinux_2_18_x86_64", - }, - }, - ) - -_tests.append(_test_cp37_abi3_manylinux_2_17_x86_64) - -def _test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64(env): - # I've seen such a wheel being built for `uv` - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-cp37-manylinux_2_17_arm64.musllinux_1_1_arm64.whl", - python_version = "3.7", - glibc_versions = [ - (2, 16), - (2, 17), - (2, 18), - ], - muslc_versions = [ - (1, 1), - ], - want = [], - want_versions = { - ":is_cp37_cp37_manylinux_aarch64": { - (2, 17): ":is_cp37_cp37_manylinux_2_17_aarch64", - (2, 18): ":is_cp37_cp37_manylinux_2_18_aarch64", - }, - ":is_cp37_cp37_musllinux_aarch64": { - (1, 1): ":is_cp37_cp37_musllinux_1_1_aarch64", - }, - }, - ) - -_tests.append(_test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64) - def render_pkg_aliases_test_suite(name): """Create the test suite. From 818f3e3079eb26b9b4c8a23d813e0561033c098e Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 18:56:45 +0900 Subject: [PATCH 05/29] wip: add notes fix bugs and address comments --- MODULE.bazel | 21 ++++++- python/private/pypi/extension.bzl | 64 ++++++++++++-------- python/private/pypi/pep508_env.bzl | 74 +++++++++++------------- tests/pypi/extension/extension_tests.bzl | 16 ++--- 4 files changed, 99 insertions(+), 76 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 9c9ed9d19a..aecc5064f7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -74,7 +74,12 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") env = {"platform_version": "0"}, os_name = "linux", platform = "linux_{}".format(cpu), - platform_tags = [ + whl_abi_tags = [ + "cp{major}{minor}", + "abi3", + "none", + ], + whl_platform_tags = [ "linux_*_{}".format(cpu), "manylinux_*_{}".format(cpu), ], @@ -103,7 +108,12 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") env = {"platform_version": "14.0"}, os_name = "osx", platform = "osx_{}".format(cpu), - platform_tags = [ + whl_abi_tags = [ + "cp{major}{minor}", + "abi3", + "none", + ], + whl_platform_tags = [ "macosx_*_{}".format(suffix) for suffix in platform_tag_cpus ], @@ -129,7 +139,12 @@ pip.default( env = {"platform_version": "0"}, os_name = "windows", platform = "windows_x86_64", - platform_tags = ["win_amd64"], + whl_abi_tags = [ + "cp{major}{minor}", + "abi3", + "none", + ], + whl_platform_tags = ["win_amd64"], ) pip.parse( # NOTE @aignas 2024-10-26: We have an integration test that depends on us diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 3e83734e8a..78a39ec5d7 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -68,14 +68,23 @@ def _whl_mods_impl(whl_mods_dict): def _platforms(*, python_version, minor_mapping, config): platforms = {} - python_version = full_version( - version = python_version, - minor_mapping = minor_mapping, + python_version = version.parse( + full_version( + version = python_version, + minor_mapping = minor_mapping, + ), + strict = True, ) for platform, values in config.platforms.items(): + # TODO @aignas 2025-07-07: fix the logic here implementation = values.env["implementation_name"][:2].lower() - abi = "{}3{}".format(implementation, python_version[2:]) + + # TODO @aignas 2025-07-07: move the abi construction somewhere else + abi = "{impl}{0}{1}.{2}".format( + impl = implementation, + *python_version.release + ) key = "{}_{}".format(abi, platform) env_ = env(struct( @@ -86,10 +95,13 @@ def _platforms(*, python_version, minor_mapping, config): platforms[key] = struct( env = env_, want_abis = [ - v.format(*python_version.split(".")) - for v in values.want_abis + v.format( + major = python_version.release[0], + minor = python_version.release[1], + ) + for v in values.whl_abi_tags ], - platform_tags = values.platform_tags, + platform_tags = values.whl_platform_tags, ) return platforms @@ -385,10 +397,10 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net ), ) -def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, want_abis, platform_tags, override = False): +def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, whl_abi_tags, whl_platform_tags, override = False): """Set the value in the config if the value is provided""" config.setdefault("platforms", {}) - if platform and (os_name or arch_name or config_settings or platform_tags or env): + if platform and (os_name or arch_name or config_settings or whl_platform_tags or env): if not override and config.get("platforms", {}).get(platform): return @@ -402,21 +414,21 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = { if not arch_name: fail("'arch_name' is required") - if platform_tags and "any" not in platform_tags: + if whl_platform_tags and "any" not in whl_platform_tags: # the lowest priority one needs to be the first one - platform_tags = ["any"] + platform_tags + whl_platform_tags = ["any"] + whl_platform_tags config["platforms"][platform] = struct( name = platform.replace("-", "_").lower(), os_name = os_name, arch_name = arch_name, config_settings = config_settings, - want_abis = want_abis or [ - "cp{0}{1}", + whl_abi_tags = whl_abi_tags or [ + "cp{major}{minor}", "abi3", "none", ], - platform_tags = platform_tags, + whl_platform_tags = whl_platform_tags, env = { # default to this "implementation_name": "cpython", @@ -491,8 +503,8 @@ You cannot use both the additive_build_content and additive_build_content_file a env = tag.env, os_name = tag.os_name, platform = tag.platform, - platform_tags = tag.platform_tags, - want_abis = tag.want_abis, + whl_abi_tags = tag.whl_abi_tags, + whl_platform_tags = tag.whl_platform_tags, override = mod.is_root, # TODO @aignas 2025-05-19: add more attr groups: # * for AUTH - the default `netrc` usage could be configured through a common @@ -844,20 +856,24 @@ If you are defining custom platforms in your project and don't want things to cl [isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate """, ), - "platform_tags": attr.string_list( + "whl_abi_tags": attr.string_list( + doc = """\ +A list of ABIs to select wheels for. The values can be either strings or include template +parameters like `{0}` which will be replaced with python version parts. e.g. `cp{0}{1}` will +result in `cp313` given the full python version is `3.13.5`. + +See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#abi-tag) for more information. +""", + ), + "whl_platform_tags": attr.string_list( doc = """\ A list of `platform_tag` matchers so that we can select the best wheel based on the user preference. Per platform we will select a single wheel and the last match from this list will take precedence. The items in this list can contain a single `*` character that is equivalent to `.*` regex match. -""", - ), - "want_abis": attr.string_list( - doc = """\ -A list of ABIs to select wheels for. The values can be either strings or include template -parameters like `{0}` which will be replaced with python version parts. e.g. `cp{0}{1}` will -result in `cp313` given the full python version is `3.13.5`. + +See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag) for more information. """, ), } diff --git a/python/private/pypi/pep508_env.bzl b/python/private/pypi/pep508_env.bzl index c2d404bc3e..3717c68398 100644 --- a/python/private/pypi/pep508_env.bzl +++ b/python/private/pypi/pep508_env.bzl @@ -15,6 +15,8 @@ """This module is for implementing PEP508 environment definition. """ +_DEFAULT = "//conditions:default" + # See https://stackoverflow.com/a/45125525 platform_machine_aliases = { # These pairs mean the same hardware, but different values may be used @@ -59,11 +61,11 @@ platform_machine_select_map = { "@platforms//cpu:x86_64": "x86_64", # The value is empty string if it cannot be determined: # https://docs.python.org/3/library/platform.html#platform.machine - "//conditions:default": "", + _DEFAULT: "", } # Platform system returns results from the `uname` call. -_platform_system_values = { +platform_system_select_map = { # See https://peps.python.org/pep-0738/#platform "android": "Android", "freebsd": "FreeBSD", @@ -75,15 +77,9 @@ _platform_system_values = { "openbsd": "OpenBSD", "osx": "Darwin", "windows": "Windows", -} - -platform_system_select_map = { - "@platforms//os:{}".format(bazel_os): py_system - for bazel_os, py_system in _platform_system_values.items() -} | { # The value is empty string if it cannot be determined: # https://docs.python.org/3/library/platform.html#platform.machine - "//conditions:default": "", + _DEFAULT: "", } # The copy of SO [answer](https://stackoverflow.com/a/13874620) containing @@ -111,30 +107,24 @@ platform_system_select_map = { # (**) Prior Python 3.8 could also be aix5 or aix7; use sys.platform.startswith() # # We are using only the subset that we actually support. -_sys_platform_values = { +sys_platform_select_map = { # These values are decided by the sys.platform docs. - "android": "android", - "emscripten": "emscripten", + "@platforms//os:android": "android", + "@platforms//os:emscripten": "emscripten", # NOTE: The below values are approximations. The sys.platform() docs # don't have documented values for these OSes. Per docs, the # sys.platform() value reflects the OS at the time Python was *built* # instead of the runtime (target) OS value. - "freebsd": "freebsd", - "ios": "ios", - "linux": "linux", - "openbsd": "openbsd", - "osx": "darwin", - "wasi": "wasi", - "windows": "win32", -} - -sys_platform_select_map = { - "@platforms//os:{}".format(bazel_os): py_platform - for bazel_os, py_platform in _sys_platform_values.items() -} | { + "@platforms//os:freebsd": "freebsd", + "@platforms//os:ios": "ios", + "@platforms//os:linux": "linux", + "@platforms//os:openbsd": "openbsd", + "@platforms//os:osx": "darwin", + "@platforms//os:wasi": "wasi", + "@platforms//os:windows": "win32", # For lack of a better option, use empty string. No standard doc/spec # about sys_platform value. - "//conditions:default": "", + _DEFAULT: "", } # The "java" value is documented, but with Jython defunct, @@ -142,19 +132,18 @@ sys_platform_select_map = { # The os.name value is technically a property of the runtime, not the # targetted runtime OS, but the distinction shouldn't matter if # things are properly configured. -_os_name_values = { - "linux": "posix", - "osx": "posix", - "windows": "nt", -} - os_name_select_map = { - "@platforms//os:{}".format(bazel_os): py_os - for bazel_os, py_os in _os_name_values.items() -} | { - "//conditions:default": "posix", + "@platforms//os:windows": "nt", + _DEFAULT: "posix", } +# TODO @aignas 2025-07-07: add a test to ensure that this is tested +def _get_from_map(m, key): + if _DEFAULT in m: + return m.get(key, m[_DEFAULT]) + else: + return m[key] + def env(target_platform, *, extra = None): """Return an env target platform @@ -174,6 +163,7 @@ def env(target_platform, *, extra = None): env["extra"] = extra if target_platform.abi: + # TODO @aignas 2025-07-07: do not parse the version like this here minor_version, _, micro_version = target_platform.abi[3:].partition(".") micro_version = micro_version or "0" env = env | { @@ -181,13 +171,15 @@ def env(target_platform, *, extra = None): "python_full_version": "3.{}.{}".format(minor_version, micro_version), "python_version": "3.{}".format(minor_version), } + if target_platform.os and target_platform.arch: - os = target_platform.os + os = "@platforms//os:{}".format(target_platform.os) + arch = "@platforms//cpu:{}".format(target_platform.arch) env = env | { - "os_name": _os_name_values.get(os, ""), - "platform_machine": target_platform.arch, - "platform_system": _platform_system_values.get(os, ""), - "sys_platform": _sys_platform_values.get(os, ""), + "os_name": _get_from_map(os_name_select_map, os), + "platform_machine": _get_from_map(platform_machine_select_map, arch), + "platform_system": _get_from_map(platform_system_select_map, os), + "sys_platform": _get_from_map(sys_platform_select_map, os), } set_missing_env_defaults(env) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index ec6b08eb60..533cb89360 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -65,9 +65,9 @@ def _mod(*, name, default = [], parse = [], override = [], whl_mods = [], is_roo "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), ], - platform_tags = platform_tags, + whl_platform_tags = whl_platform_tags, ) - for (os, cpu), platform_tags in { + for (os, cpu), whl_platform_tags in { ("linux", "x86_64"): ["linux_*_x86_64", "manylinux_*_x86_64"], ("linux", "aarch64"): ["linux_*_aarch64", "manylinux_*_aarch64"], ("osx", "aarch64"): ["macosx_*_arm64"], @@ -99,17 +99,17 @@ def _default( config_settings = None, os_name = None, platform = None, - platform_tags = None, + whl_platform_tags = None, env = None, - want_abis = None): + whl_abi_tags = None): return struct( arch_name = arch_name, os_name = os_name, platform = platform, - platform_tags = platform_tags or [], + whl_platform_tags = whl_platform_tags or [], config_settings = config_settings, env = env or {}, - want_abis = want_abis or [], + whl_abi_tags = whl_abi_tags or [], ) def _parse( @@ -407,9 +407,9 @@ def _test_torch_experimental_index_url(env): "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), ], - platform_tags = platform_tags, + whl_platform_tags = whl_platform_tags, ) - for (os, cpu), platform_tags in { + for (os, cpu), whl_platform_tags in { ("linux", "x86_64"): ["linux_*_x86_64", "manylinux_*_x86_64"], ("linux", "aarch64"): ["linux_*_aarch64", "manylinux_*_aarch64"], ("osx", "aarch64"): ["macosx_*_arm64"], From b00d7a2f56ad61bea67e0d8e6db220d0b4d081e9 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 21:35:31 +0900 Subject: [PATCH 06/29] extra fixes and add a helper for python_tag --- python/private/pypi/BUILD.bazel | 7 +++++++ python/private/pypi/extension.bzl | 4 ++-- python/private/pypi/pep508_env.bzl | 16 ++++++++-------- python/private/pypi/python_tag.bzl | 23 +++++++++++++++++++++++ python/private/pypi/select_whl.bzl | 16 +++------------- 5 files changed, 43 insertions(+), 23 deletions(-) create mode 100644 python/private/pypi/python_tag.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 982c64436e..bfe15c1bc5 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -116,6 +116,7 @@ bzl_library( ":parse_whl_name_bzl", ":pep508_env_bzl", ":pip_repository_attrs_bzl", + ":python_tag_bzl", ":simpleapi_download_bzl", ":whl_config_setting_bzl", ":whl_library_bzl", @@ -337,6 +338,11 @@ bzl_library( ], ) +bzl_library( + name = "python_tag_bzl", + srcs = ["python_tag.bzl"], +) + bzl_library( name = "render_pkg_aliases_bzl", srcs = ["render_pkg_aliases.bzl"], @@ -363,6 +369,7 @@ bzl_library( srcs = ["select_whl.bzl"], deps = [ ":parse_whl_name_bzl", + ":python_tag_bzl", "//python/private:version_bzl", ], ) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 78a39ec5d7..592fe238fd 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -31,6 +31,7 @@ load(":parse_requirements.bzl", "parse_requirements") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") load(":pip_repository_attrs.bzl", "ATTRS") +load(":python_tag.bzl", "python_tag") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":simpleapi_download.bzl", "simpleapi_download") load(":whl_config_setting.bzl", "whl_config_setting") @@ -77,8 +78,7 @@ def _platforms(*, python_version, minor_mapping, config): ) for platform, values in config.platforms.items(): - # TODO @aignas 2025-07-07: fix the logic here - implementation = values.env["implementation_name"][:2].lower() + implementation = python_tag(values.env["implementation_name"]) # TODO @aignas 2025-07-07: move the abi construction somewhere else abi = "{impl}{0}{1}.{2}".format( diff --git a/python/private/pypi/pep508_env.bzl b/python/private/pypi/pep508_env.bzl index 3717c68398..4445df4c9f 100644 --- a/python/private/pypi/pep508_env.bzl +++ b/python/private/pypi/pep508_env.bzl @@ -67,16 +67,16 @@ platform_machine_select_map = { # Platform system returns results from the `uname` call. platform_system_select_map = { # See https://peps.python.org/pep-0738/#platform - "android": "Android", - "freebsd": "FreeBSD", + "@platforms//os:android": "Android", + "@platforms//os:freebsd": "FreeBSD", # See https://peps.python.org/pep-0730/#platform # NOTE: Per Pep 730, "iPadOS" is also an acceptable value - "ios": "iOS", - "linux": "Linux", - "netbsd": "NetBSD", - "openbsd": "OpenBSD", - "osx": "Darwin", - "windows": "Windows", + "@platforms//os:ios": "iOS", + "@platforms//os:linux": "Linux", + "@platforms//os:netbsd": "NetBSD", + "@platforms//os:openbsd": "OpenBSD", + "@platforms//os:osx": "Darwin", + "@platforms//os:windows": "Windows", # The value is empty string if it cannot be determined: # https://docs.python.org/3/library/platform.html#platform.machine _DEFAULT: "", diff --git a/python/private/pypi/python_tag.bzl b/python/private/pypi/python_tag.bzl new file mode 100644 index 0000000000..4c2197fbe3 --- /dev/null +++ b/python/private/pypi/python_tag.bzl @@ -0,0 +1,23 @@ +"A simple utility function to get the python_tag from the implementation name" + +# Taken from +# https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#python-tag +_PY_TAGS = { + # "py": Generic Python (does not require implementation-specific features) + "cpython": "cp", + "ironpython": "ip", + "jython": "jy", + "pypy": "pp", +} +PY_TAG_GENERIC = "py" + +def python_tag(implementation_name): + """Get the python_tag from the implementation_name. + + Args: + implementation_name: {type}`str` the implementation name, e.g. "cpython" + + Returns: + A {type}`str` that represents the python_tag. + """ + return _PY_TAGS.get(implementation_name, implementation_name) diff --git a/python/private/pypi/select_whl.bzl b/python/private/pypi/select_whl.bzl index 3d8a90235f..3843e80739 100644 --- a/python/private/pypi/select_whl.bzl +++ b/python/private/pypi/select_whl.bzl @@ -2,17 +2,7 @@ load("//python/private:version.bzl", "version") load(":parse_whl_name.bzl", "parse_whl_name") - -# Taken from -# https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#python-tag -_PY_TAGS = { - # "py": Generic Python (does not require implementation-specific features) - "cpython": "cp", - "ironpython": "ip", - "jython": "jy", - "pypy": "pp", -} -_PY = "py" +load(":python_tag.bzl", "PY_TAG_GENERIC", "python_tag") def _get_priority(*, tag, values, allow_wildcard = True): for priority, wp in enumerate(values): @@ -51,12 +41,12 @@ def select_whl(*, whls, python_version, platforms, want_abis, implementation_nam """ py_version = version.parse(python_version, strict = True) candidates = {} - implementation = _PY_TAGS.get(implementation_name, implementation_name) + implementation = python_tag(implementation_name) for whl in whls: parsed = parse_whl_name(whl.filename) - if parsed.python_tag.startswith(_PY): + if parsed.python_tag.startswith(PY_TAG_GENERIC): pass elif not parsed.python_tag.startswith(implementation): if logger: From bb838c031ecfd183970dc9fee5da0cf40182c4bd Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 21:56:22 +0900 Subject: [PATCH 07/29] more cleanup --- python/private/pypi/BUILD.bazel | 8 +-- python/private/pypi/extension.bzl | 27 ++++----- python/private/pypi/pep508_env.bzl | 31 +++++----- python/private/pypi/pep508_platform.bzl | 57 ------------------- python/private/pypi/python_tag.bzl | 24 +++++++- .../parse_requirements_tests.bzl | 13 ++++- tests/pypi/pep508/evaluate_tests.bzl | 14 +++-- 7 files changed, 70 insertions(+), 104 deletions(-) delete mode 100644 python/private/pypi/pep508_platform.bzl diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index bfe15c1bc5..f15135f5fc 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -263,11 +263,6 @@ bzl_library( ], ) -bzl_library( - name = "pep508_platform_bzl", - srcs = ["pep508_platform.bzl"], -) - bzl_library( name = "pep508_requirement_bzl", srcs = ["pep508_requirement.bzl"], @@ -341,6 +336,9 @@ bzl_library( bzl_library( name = "python_tag_bzl", srcs = ["python_tag.bzl"], + deps = [ + "//python/private:version_bzl", + ], ) bzl_library( diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 592fe238fd..f19feacf2c 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -78,22 +78,17 @@ def _platforms(*, python_version, minor_mapping, config): ) for platform, values in config.platforms.items(): - implementation = python_tag(values.env["implementation_name"]) - - # TODO @aignas 2025-07-07: move the abi construction somewhere else - abi = "{impl}{0}{1}.{2}".format( - impl = implementation, - *python_version.release - ) - key = "{}_{}".format(abi, platform) - - env_ = env(struct( - abi = abi, - os = values.os_name, - arch = values.arch_name, - )) | values.env - platforms[key] = struct( - env = env_, + # TODO @aignas 2025-07-07: this is probably doing the parsing of the version too + # many times. + pytag = python_tag(values.env["implementation_name"], python_version.string) + + platforms["{}.{}_{}".format(pytag, python_version.release[2], platform)] = struct( + env = env( + env = values.env, + os = values.os_name, + arch = values.arch_name, + python_version = python_version.string, + ), want_abis = [ v.format( major = python_version.release[0], diff --git a/python/private/pypi/pep508_env.bzl b/python/private/pypi/pep508_env.bzl index 4445df4c9f..071c3acf47 100644 --- a/python/private/pypi/pep508_env.bzl +++ b/python/private/pypi/pep508_env.bzl @@ -15,6 +15,8 @@ """This module is for implementing PEP508 environment definition. """ +load("//python/private:version.bzl", "version") + _DEFAULT = "//conditions:default" # See https://stackoverflow.com/a/45125525 @@ -144,37 +146,38 @@ def _get_from_map(m, key): else: return m[key] -def env(target_platform, *, extra = None): +def env(*, env = None, os, arch, python_version = "", extra = None): """Return an env target platform NOTE: This is for use during the loading phase. For the analysis phase, `env_marker_setting()` constructs the env dict. Args: - target_platform: {type}`str` the target platform identifier, e.g. - `cp33_linux_aarch64` + env: {type}`str` the environment. + os: {type}`str` the OS name. + arch: {type}`str` the CPU name. + python_version: {type}`str` the full python version. extra: {type}`str` the extra value to be added into the env. Returns: A dict that can be used as `env` in the marker evaluation. """ - env = create_env() + env = env or {} + env = env | create_env() if extra != None: env["extra"] = extra - if target_platform.abi: - # TODO @aignas 2025-07-07: do not parse the version like this here - minor_version, _, micro_version = target_platform.abi[3:].partition(".") - micro_version = micro_version or "0" + if python_version: + v = version.parse(python_version) env = env | { - "implementation_version": "3.{}.{}".format(minor_version, micro_version), - "python_full_version": "3.{}.{}".format(minor_version, micro_version), - "python_version": "3.{}".format(minor_version), + "implementation_version": "{}.{}.{}".format(v.release[0], v.release[1], v.release[2]), + "python_full_version": "{}.{}.{}".format(v.release[0], v.release[1], v.release[2]), + "python_version": "{}.{}".format(v.release[0], v.release[1]), } - if target_platform.os and target_platform.arch: - os = "@platforms//os:{}".format(target_platform.os) - arch = "@platforms//cpu:{}".format(target_platform.arch) + if os and arch: + os = "@platforms//os:{}".format(os) + arch = "@platforms//cpu:{}".format(arch) env = env | { "os_name": _get_from_map(os_name_select_map, os), "platform_machine": _get_from_map(platform_machine_select_map, arch), diff --git a/python/private/pypi/pep508_platform.bzl b/python/private/pypi/pep508_platform.bzl deleted file mode 100644 index 381a8d7a08..0000000000 --- a/python/private/pypi/pep508_platform.bzl +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2025 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""The platform abstraction -""" - -def platform(*, abi = None, os = None, arch = None): - """platform returns a struct for the platform. - - Args: - abi: {type}`str | None` the target ABI, e.g. `"cp39"`. - os: {type}`str | None` the target os, e.g. `"linux"`. - arch: {type}`str | None` the target CPU, e.g. `"aarch64"`. - - Returns: - A struct. - """ - - # Note, this is used a lot as a key in dictionaries, so it cannot contain - # methods. - return struct( - abi = abi, - os = os, - arch = arch, - ) - -def platform_from_str(p, python_version): - """Return a platform from a string. - - Args: - p: {type}`str` the actual string. - python_version: {type}`str` the python version to add to platform if needed. - - Returns: - A struct that is returned by the `_platform` function. - """ - if p.startswith("cp"): - abi, _, p = p.partition("_") - elif python_version: - major, _, tail = python_version.partition(".") - abi = "cp{}{}".format(major, tail) - else: - abi = None - - os, _, arch = p.partition("_") - return platform(abi = abi, os = os or None, arch = arch or None) diff --git a/python/private/pypi/python_tag.bzl b/python/private/pypi/python_tag.bzl index 4c2197fbe3..224c5f96f0 100644 --- a/python/private/pypi/python_tag.bzl +++ b/python/private/pypi/python_tag.bzl @@ -1,5 +1,7 @@ "A simple utility function to get the python_tag from the implementation name" +load("//python/private:version.bzl", "version") + # Taken from # https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#python-tag _PY_TAGS = { @@ -8,16 +10,32 @@ _PY_TAGS = { "ironpython": "ip", "jython": "jy", "pypy": "pp", + "python": "py", } PY_TAG_GENERIC = "py" -def python_tag(implementation_name): +def python_tag(implementation_name, python_version = ""): """Get the python_tag from the implementation_name. Args: implementation_name: {type}`str` the implementation name, e.g. "cpython" + python_version: {type}`str` a version who can be parsed using PEP440 compliant + parser. Returns: - A {type}`str` that represents the python_tag. + A {type}`str` that represents the python_tag with a version if the + python_version is given. """ - return _PY_TAGS.get(implementation_name, implementation_name) + if python_version: + v = version.parse(python_version, strict = True) + suffix = "{}{}".format( + v.release[0], + v.release[1] if len(v.release) > 1 else "", + ) + else: + suffix = "" + + return "{}{}".format( + _PY_TAGS.get(implementation_name, implementation_name), + suffix, + ) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index d862d0ebde..6c63355aa5 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -17,7 +17,6 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private/pypi:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility -load("//python/private/pypi:pep508_platform.bzl", "platform_from_str") # buildifier: disable=bzl-visibility def _mock_ctx(): testdata = { @@ -527,12 +526,20 @@ def _test_overlapping_shas_with_index_results(env): platforms = { "cp39_linux_x86_64": struct( platform_tags = ["any"], - env = pep508_env(platform_from_str("cp39_linux_x86_64", "")), + env = pep508_env( + python_version = "3.9.0", + os = "linux", + arch = "x86_64", + ), want_abis = ["none"], ), "cp39_osx_x86_64": struct( platform_tags = ["macosx_*"], - env = pep508_env(platform_from_str("cp39_linux_x86_64", "")), + env = pep508_env( + python_version = "3.9.0", + os = "osx", + arch = "x86_64", + ), want_abis = ["none"], ), }, diff --git a/tests/pypi/pep508/evaluate_tests.bzl b/tests/pypi/pep508/evaluate_tests.bzl index cc867f346c..d2325b32a4 100644 --- a/tests/pypi/pep508/evaluate_tests.bzl +++ b/tests/pypi/pep508/evaluate_tests.bzl @@ -16,7 +16,6 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility load("//python/private/pypi:pep508_evaluate.bzl", "evaluate", "tokenize") # buildifier: disable=bzl-visibility -load("//python/private/pypi:pep508_platform.bzl", "platform_from_str") # buildifier: disable=bzl-visibility _tests = [] @@ -244,18 +243,18 @@ _tests.append(_evaluate_partial_only_extra) def _evaluate_with_aliases(env): # When - for target_platform, tests in { + for (os, cpu), tests in { # buildifier: @unsorted-dict-items - "osx_aarch64": { + ("osx", "aarch64"): { "platform_system == 'Darwin' and platform_machine == 'arm64'": True, "platform_system == 'Darwin' and platform_machine == 'aarch64'": True, "platform_system == 'Darwin' and platform_machine == 'amd64'": False, }, - "osx_x86_64": { + ("osx", "x86_64"): { "platform_system == 'Darwin' and platform_machine == 'amd64'": True, "platform_system == 'Darwin' and platform_machine == 'x86_64'": True, }, - "osx_x86_32": { + ("osx", "x86_32"): { "platform_system == 'Darwin' and platform_machine == 'i386'": True, "platform_system == 'Darwin' and platform_machine == 'i686'": True, "platform_system == 'Darwin' and platform_machine == 'x86_32'": True, @@ -263,7 +262,10 @@ def _evaluate_with_aliases(env): }, }.items(): # buildifier: @unsorted-dict-items for input, want in tests.items(): - _check_evaluate(env, input, want, pep508_env(platform_from_str(target_platform, ""))) + _check_evaluate(env, input, want, pep508_env( + os = os, + arch = cpu, + )) _tests.append(_evaluate_with_aliases) From eb2de67884751d34f7b79763730992459c4a1e1f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 22:00:48 +0900 Subject: [PATCH 08/29] clarify the doc --- python/private/pypi/extension.bzl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index f19feacf2c..9a3cfb81c8 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -80,9 +80,15 @@ def _platforms(*, python_version, minor_mapping, config): for platform, values in config.platforms.items(): # TODO @aignas 2025-07-07: this is probably doing the parsing of the version too # many times. - pytag = python_tag(values.env["implementation_name"], python_version.string) + key = "{}{}{}.{}_{}".format( + python_tag(values.env["implementation_name"]), + python_version.release[0], + python_version.release[1], + python_version.release[2], + platform, + ) - platforms["{}.{}_{}".format(pytag, python_version.release[2], platform)] = struct( + platforms[key] = struct( env = env( env = values.env, os = values.os_name, @@ -795,6 +801,7 @@ _default_attrs = { "arch_name": attr.string( doc = """\ The CPU architecture name to be used. +You can use any cpu name from the `@platforms//cpu:` package. :::{note} Either this or {attr}`env` `platform_machine` key should be specified. @@ -836,6 +843,7 @@ This is only used if the {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` is enabled. "os_name": attr.string( doc = """\ The OS name to be used. +You can use any OS name from the `@platforms//os:` package. :::{note} Either this or the appropriate `env` keys should be specified. From 233ca9df44643317a91888c859b790cb0dcc6764 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 22:03:44 +0900 Subject: [PATCH 09/29] rename --- .bazelrc | 4 ++-- examples/bzlmod/MODULE.bazel | 2 +- python/private/pypi/extension.bzl | 2 +- python/private/pypi/parse_requirements.bzl | 2 +- python/private/pypi/select_whl.bzl | 8 ++++---- .../parse_requirements_tests.bzl | 4 ++-- tests/pypi/select_whl/select_whl_tests.bzl | 18 +++++++++--------- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bazelrc b/.bazelrc index d7e1771336..8997db9f91 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg test --test_output=errors diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 6b1ee2c351..582dbf3bd1 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -172,7 +172,7 @@ pip.default( os_name = "windows", platform = "windows_aarch64", platform_tags = ["win_amd64"], - want_abis = [], # default to all ABIs + whl_abi_tags = [], # default to all ABIs ) # To fetch pip dependencies, use pip.parse. We can pass in various options, diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 9a3cfb81c8..3972d330c3 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -95,7 +95,7 @@ def _platforms(*, python_version, minor_mapping, config): arch = values.arch_name, python_version = python_version.string, ), - want_abis = [ + whl_abi_tags = [ v.format( major = python_version.release[0], minor = python_version.release[1], diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 6f9ae56fc7..e769279a65 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -419,7 +419,7 @@ def _add_dists(*, requirement, index_urls, target_platform, logger = None): whls = whls, python_version = target_platform.env["python_full_version"], implementation_name = target_platform.env["implementation_name"], - want_abis = target_platform.want_abis, + whl_abi_tags = target_platform.whl_abi_tags, platforms = target_platform.platform_tags, logger = logger, ) or sdist diff --git a/python/private/pypi/select_whl.bzl b/python/private/pypi/select_whl.bzl index 3843e80739..30d803956f 100644 --- a/python/private/pypi/select_whl.bzl +++ b/python/private/pypi/select_whl.bzl @@ -20,7 +20,7 @@ def _get_priority(*, tag, values, allow_wildcard = True): return None -def select_whl(*, whls, python_version, platforms, want_abis, implementation_name = "cpython", limit = 1, logger = None): +def select_whl(*, whls, python_version, platforms, whl_abi_tags, implementation_name = "cpython", limit = 1, logger = None): """Select a whl that is the most suitable for the given platform. Args: @@ -30,7 +30,7 @@ def select_whl(*, whls, python_version, platforms, want_abis, implementation_nam platforms: {type}`list[str]` the target platform identifiers that may contain a single `*` character. implementation_name: {type}`str` the `implementation_name` from the target_platform env. - want_abis: {type}`str` the ABIs that the target_platform is compatible with. + whl_abi_tags: {type}`str` the ABIs that the target_platform is compatible with. limit: {type}`int` number of wheels to return. Defaults to 1. logger: {type}`struct` the logger instance. @@ -76,14 +76,14 @@ def select_whl(*, whls, python_version, platforms, want_abis, implementation_nam abi_priority = _get_priority( tag = parsed.abi_tag, - values = want_abis, + values = whl_abi_tags, allow_wildcard = False, ) if abi_priority == None: if logger: logger.debug(lambda: "The abi '{}' does not match given list: {}".format( parsed.abi_tag, - want_abis, + whl_abi_tags, )) continue platform_priority = _get_priority( diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 6c63355aa5..aa6818e51a 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -531,7 +531,7 @@ def _test_overlapping_shas_with_index_results(env): os = "linux", arch = "x86_64", ), - want_abis = ["none"], + whl_abi_tags = ["none"], ), "cp39_osx_x86_64": struct( platform_tags = ["macosx_*"], @@ -540,7 +540,7 @@ def _test_overlapping_shas_with_index_results(env): os = "osx", arch = "x86_64", ), - want_abis = ["none"], + whl_abi_tags = ["none"], ), }, get_index_urls = lambda _, __: { diff --git a/tests/pypi/select_whl/select_whl_tests.bzl b/tests/pypi/select_whl/select_whl_tests.bzl index 57ff39e76a..4269680ef8 100644 --- a/tests/pypi/select_whl/select_whl_tests.bzl +++ b/tests/pypi/select_whl/select_whl_tests.bzl @@ -106,7 +106,7 @@ def _test_simplest(env): got = _select_whl( whls = whls, platforms = ["any"], - want_abis = ["abi3"], + whl_abi_tags = ["abi3"], python_version = "3.0", ) _match( @@ -131,7 +131,7 @@ def _test_select_by_supported_py_version(env): got = _select_whl( whls = whls, platforms = ["any"], - want_abis = ["abi3"], + whl_abi_tags = ["abi3"], python_version = "3.{}".format(minor_version), ) _match(env, [got], match) @@ -153,7 +153,7 @@ def _test_select_by_supported_cp_version(env): got = _select_whl( whls = whls, platforms = ["any"], - want_abis = ["abi3"], + whl_abi_tags = ["abi3"], python_version = "3.{}".format(minor_version), ) _match(env, [got], match) @@ -175,7 +175,7 @@ def _test_supported_cp_version_manylinux(env): got = _select_whl( whls = whls, platforms = ["manylinux_x86_64"], - want_abis = ["none"], + whl_abi_tags = ["none"], python_version = "3.{}".format(minor_version), ) _match(env, [got], match) @@ -187,7 +187,7 @@ def _test_ignore_unsupported(env): got = _select_whl( whls = whls, platforms = ["any"], - want_abis = ["none"], + whl_abi_tags = ["none"], python_version = "3.0", ) if got: @@ -205,7 +205,7 @@ def _test_match_abi_and_not_py_version(env): got = _select_whl( whls = whls, platforms = platforms, - want_abis = ["abi3", "cp37m"], + whl_abi_tags = ["abi3", "cp37m"], python_version = "3.7", ) _match( @@ -217,7 +217,7 @@ def _test_match_abi_and_not_py_version(env): got = _select_whl( whls = whls, platforms = platforms[::-1], - want_abis = ["abi3", "cp37m"], + whl_abi_tags = ["abi3", "cp37m"], python_version = "3.7", ) _match( @@ -237,7 +237,7 @@ def _test_select_filename_with_many_tags(env): "musllinux_*_i686", "manylinux_*_i686", ], - want_abis = ["none", "abi3", "cp39"], + whl_abi_tags = ["none", "abi3", "cp39"], python_version = "3.9", limit = 5, ) @@ -261,7 +261,7 @@ def _test_freethreaded_wheels(env): "any", "musllinux_*_x86_64", ], - want_abis = ["none", "abi3", "cp313", "cp313t"], + whl_abi_tags = ["none", "abi3", "cp313", "cp313t"], python_version = "3.13", limit = 8, ) From 0991ad111dea06df5cd85136145bcaf1c037c8ab Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 22:06:33 +0900 Subject: [PATCH 10/29] remove merge conflicts --- python/private/pypi/extension.bzl | 6 ------ 1 file changed, 6 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 3972d330c3..4f284f8c38 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -409,12 +409,6 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = { if key not in _SUPPORTED_PEP508_KEYS: fail("Unsupported key in the PEP508 environment: {}".format(key)) - if not os_name: - fail("'os_name' is required") - - if not arch_name: - fail("'arch_name' is required") - if whl_platform_tags and "any" not in whl_platform_tags: # the lowest priority one needs to be the first one whl_platform_tags = ["any"] + whl_platform_tags From 95535027c06c14d1d8655fc6ad2fe2b1eb8d54de Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 22:10:30 +0900 Subject: [PATCH 11/29] add a test for a bug --- python/private/pypi/pep508_env.bzl | 1 - tests/pypi/pep508/evaluate_tests.bzl | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/python/private/pypi/pep508_env.bzl b/python/private/pypi/pep508_env.bzl index 071c3acf47..dacd825380 100644 --- a/python/private/pypi/pep508_env.bzl +++ b/python/private/pypi/pep508_env.bzl @@ -139,7 +139,6 @@ os_name_select_map = { _DEFAULT: "posix", } -# TODO @aignas 2025-07-07: add a test to ensure that this is tested def _get_from_map(m, key): if _DEFAULT in m: return m.get(key, m[_DEFAULT]) diff --git a/tests/pypi/pep508/evaluate_tests.bzl b/tests/pypi/pep508/evaluate_tests.bzl index d2325b32a4..7139b0a52b 100644 --- a/tests/pypi/pep508/evaluate_tests.bzl +++ b/tests/pypi/pep508/evaluate_tests.bzl @@ -260,6 +260,13 @@ def _evaluate_with_aliases(env): "platform_system == 'Darwin' and platform_machine == 'x86_32'": True, "platform_system == 'Darwin' and platform_machine == 'x86_64'": False, }, + ("freebsd", "x86_32"): { + "platform_system == 'FreeBSD' and platform_machine == 'i386'": True, + "platform_system == 'FreeBSD' and platform_machine == 'i686'": True, + "platform_system == 'FreeBSD' and platform_machine == 'x86_32'": True, + "platform_system == 'FreeBSD' and platform_machine == 'x86_64'": False, + "platform_system == 'FreeBSD' and os_name == 'posix'": True, + }, }.items(): # buildifier: @unsorted-dict-items for input, want in tests.items(): _check_evaluate(env, input, want, pep508_env( From a142c63eab5836b1fbf20cbef7461e308add8ea8 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 22:11:45 +0900 Subject: [PATCH 12/29] simplify --- python/private/pypi/pep508_env.bzl | 9 ++++++--- tests/pypi/pep508/evaluate_tests.bzl | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/python/private/pypi/pep508_env.bzl b/python/private/pypi/pep508_env.bzl index dacd825380..a51d154219 100644 --- a/python/private/pypi/pep508_env.bzl +++ b/python/private/pypi/pep508_env.bzl @@ -168,10 +168,13 @@ def env(*, env = None, os, arch, python_version = "", extra = None): if python_version: v = version.parse(python_version) + major = v.release[0] + minor = v.release[1] + micro = v.release[2] if len(v.release) > 2 else 0 env = env | { - "implementation_version": "{}.{}.{}".format(v.release[0], v.release[1], v.release[2]), - "python_full_version": "{}.{}.{}".format(v.release[0], v.release[1], v.release[2]), - "python_version": "{}.{}".format(v.release[0], v.release[1]), + "implementation_version": "{}.{}.{}".format(major, minor, micro), + "python_full_version": "{}.{}.{}".format(major, minor, micro), + "python_version": "{}.{}".format(major, minor), } if os and arch: diff --git a/tests/pypi/pep508/evaluate_tests.bzl b/tests/pypi/pep508/evaluate_tests.bzl index 7139b0a52b..7843f88e89 100644 --- a/tests/pypi/pep508/evaluate_tests.bzl +++ b/tests/pypi/pep508/evaluate_tests.bzl @@ -272,6 +272,7 @@ def _evaluate_with_aliases(env): _check_evaluate(env, input, want, pep508_env( os = os, arch = cpu, + python_version = "3.2", )) _tests.append(_evaluate_with_aliases) From a9c2aaa295082b220a186ecab0fa3c68ca1e1c4f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 22:14:28 +0900 Subject: [PATCH 13/29] rename to whl_platform_tags --- examples/bzlmod/MODULE.bazel | 2 +- python/private/pypi/extension.bzl | 2 +- python/private/pypi/parse_requirements.bzl | 2 +- tests/pypi/parse_requirements/parse_requirements_tests.bzl | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 582dbf3bd1..95e1090f53 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -171,8 +171,8 @@ pip.default( }, os_name = "windows", platform = "windows_aarch64", - platform_tags = ["win_amd64"], whl_abi_tags = [], # default to all ABIs + whl_platform_tags = ["win_amd64"], ) # To fetch pip dependencies, use pip.parse. We can pass in various options, diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 4f284f8c38..5ecb48b062 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -102,7 +102,7 @@ def _platforms(*, python_version, minor_mapping, config): ) for v in values.whl_abi_tags ], - platform_tags = values.whl_platform_tags, + whl_platform_tags = values.whl_platform_tags, ) return platforms diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index e769279a65..557fc286ab 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -420,6 +420,6 @@ def _add_dists(*, requirement, index_urls, target_platform, logger = None): python_version = target_platform.env["python_full_version"], implementation_name = target_platform.env["implementation_name"], whl_abi_tags = target_platform.whl_abi_tags, - platforms = target_platform.platform_tags, + platforms = target_platform.whl_platform_tags, logger = logger, ) or sdist diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index aa6818e51a..f88c274ecb 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -525,22 +525,22 @@ def _test_overlapping_shas_with_index_results(env): }, platforms = { "cp39_linux_x86_64": struct( - platform_tags = ["any"], env = pep508_env( python_version = "3.9.0", os = "linux", arch = "x86_64", ), whl_abi_tags = ["none"], + whl_platform_tags = ["any"], ), "cp39_osx_x86_64": struct( - platform_tags = ["macosx_*"], env = pep508_env( python_version = "3.9.0", os = "osx", arch = "x86_64", ), whl_abi_tags = ["none"], + whl_platform_tags = ["macosx_*"], ), }, get_index_urls = lambda _, __: { From a4566b1e4fd56a76515bf11864ebbfba95ad87f2 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 22:35:44 +0900 Subject: [PATCH 14/29] remove lower tier platforms from pip.default testing --- CHANGELOG.md | 4 ++++ MODULE.bazel | 46 +++++++++++++++++++++------------------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b57af606e..81e7a5f636 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -67,6 +67,10 @@ END_UNRELEASED_TEMPLATE * 3.12.11 * 3.14.0b3 * (toolchain) Python 3.13 now references 3.13.5 +* (pypi) From now on the list of default platforms only includes `linux_x86_64`, `linux_aarch64`, + `osx_x86_64`, `osx_aarch64` and `windows_x86_64`. If you are on other platforms, you need to + use the `pip.default` to configure it yourself. If you are interested in graduating the + platform, consider helping set us up CI for them and update the documentation. {#v0-0-0-fixed} ### Fixed diff --git a/MODULE.bazel b/MODULE.bazel index aecc5064f7..8e74586041 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -87,12 +87,6 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") for cpu in [ "x86_64", "aarch64", - # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the - # `pip.default` extension. i.e. drop the below values - users will have to - # define themselves if they need them. - "arm", - "ppc", - "s390x", ] ] @@ -123,29 +117,31 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") "universal2", "arm64", ], - "x86_64": [ - "universal2", - "x86_64", + }.items() +] + +[ + pip.default( + arch_name = cpu, + config_settings = [ + "@platforms//cpu:{}".format(cpu), + "@platforms//os:windows", ], + env = {"platform_version": "0"}, + os_name = "windows", + platform = "windows_{}".format(cpu), + whl_abi_tags = [ + "cp{major}{minor}", + "abi3", + "none", + ], + whl_platform_tags = whl_platform_tags, + ) + for cpu, whl_platform_tags in { + "x86_64": ["win_amd64"], }.items() ] -pip.default( - arch_name = "x86_64", - config_settings = [ - "@platforms//cpu:x86_64", - "@platforms//os:windows", - ], - env = {"platform_version": "0"}, - os_name = "windows", - platform = "windows_x86_64", - whl_abi_tags = [ - "cp{major}{minor}", - "abi3", - "none", - ], - whl_platform_tags = ["win_amd64"], -) pip.parse( # NOTE @aignas 2024-10-26: We have an integration test that depends on us # being able to build sdists for this hub, so explicitly set this to False. From c22e3d54396f721f697fef8fe0bd218be50ed0cb Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 22:54:20 +0900 Subject: [PATCH 15/29] fix an edge case --- python/private/pypi/extension.bzl | 58 ++++++++++++++++++------ tests/pypi/extension/extension_tests.bzl | 2 + 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 5ecb48b062..0257476b4f 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -266,6 +266,10 @@ def _create_whl_repos( logger = logger, ) + use_downloader = { + normalize_name(s): False + for s in pip_attr.simpleapi_skip + } exposed_packages = {} for whl in requirements_by_platform: if whl.is_exposed: @@ -313,17 +317,26 @@ def _create_whl_repos( if v != default }) - for src in whl.srcs: + for src in sorted(whl.srcs, key = lambda x: x.filename): repo = _whl_repo( src = src, whl_library_args = whl_library_args, download_only = pip_attr.download_only, netrc = pip_attr.netrc, + # NOTE @aignas 2025-07-07: we guard against an edge-case where there + # are more platforms defined than there are wheels for and users + # disallow building from sdist. + use_downloader = use_downloader.get( + whl.name, + get_index_urls != None, # defaults to True if the get_index_urls is defined + ), auth_patterns = pip_attr.auth_patterns, python_version = major_minor, is_multiple_versions = whl.is_multiple_versions, enable_pipstar = config.enable_pipstar, ) + if repo == None: + continue repo_name = "{}_{}".format(pip_name, repo.repo_name) if repo_name in whl_libraries: @@ -342,7 +355,17 @@ def _create_whl_repos( whl_libraries = whl_libraries, ) -def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, netrc, auth_patterns, python_version, enable_pipstar = False): +def _whl_repo( + *, + src, + whl_library_args, + is_multiple_versions, + download_only, + netrc, + auth_patterns, + python_version, + use_downloader, + enable_pipstar = False): args = dict(whl_library_args) args["requirement"] = src.requirement_line is_whl = src.filename.endswith(".whl") @@ -355,19 +378,24 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net args["extra_pip_args"] = src.extra_pip_args if not src.url or (not is_whl and download_only): - # Fallback to a pip-installed wheel - target_platforms = src.target_platforms if is_multiple_versions else [] - return struct( - repo_name = pypi_repo_name( - normalize_name(src.distribution), - *target_platforms - ), - args = args, - config_setting = whl_config_setting( - version = python_version, - target_platforms = target_platforms or None, - ), - ) + if download_only and use_downloader: + # If the user did not allow using sdists and we are using the downloader + # and we are not using simpleapi_skip for this + return None + else: + # Fallback to a pip-installed wheel + target_platforms = src.target_platforms if is_multiple_versions else [] + return struct( + repo_name = pypi_repo_name( + normalize_name(src.distribution), + *target_platforms + ), + args = args, + config_setting = whl_config_setting( + version = python_version, + target_platforms = target_platforms or None, + ), + ) # This is no-op because pip is not used to download the wheel. args.pop("download_only", None) diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 533cb89360..3e73a31711 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -414,12 +414,14 @@ def _test_torch_experimental_index_url(env): ("linux", "aarch64"): ["linux_*_aarch64", "manylinux_*_aarch64"], ("osx", "aarch64"): ["macosx_*_arm64"], ("windows", "x86_64"): ["win_amd64"], + ("windows", "aarch64"): ["win_arm64"], # this should be ignored }.items() ], parse = [ _parse( hub_name = "pypi", python_version = "3.12", + download_only = True, experimental_index_url = "https://torch.index", requirements_lock = "universal.txt", ), From b40f6a4b00bfc3721493868374dce915850ddea9 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Mon, 7 Jul 2025 23:08:13 +0900 Subject: [PATCH 16/29] fix docs building --- python/private/pypi/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index f15135f5fc..2247bf04c4 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -64,6 +64,7 @@ bzl_library( deps = [ ":flags_bzl", "//python/private:flags_bzl", + "//python/private:version_bzl", "@bazel_skylib//lib:selects", ], ) From 0bd5fbf80c70c661ceb674c36305e73645893da9 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 8 Jul 2025 12:42:00 +0900 Subject: [PATCH 17/29] add back the macosx x86_64 because our CI depends on it --- MODULE.bazel | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/MODULE.bazel b/MODULE.bazel index 8e74586041..765102da90 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -117,6 +117,10 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") "universal2", "arm64", ], + "x86_64": [ + "universal2", + "x86_64", + ], }.items() ] From 95a8423d838b3e84f17f4847267542254a098582 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 8 Jul 2025 18:32:02 +0900 Subject: [PATCH 18/29] add more changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75d8445443..65e9666f79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,10 @@ END_UNRELEASED_TEMPLATE ({gh-issue}`3043`). * (pypi) The pipstar `defaults` configuration now supports any custom platform name. +* (pypi) The selection of the whls has been changed and should no longer result + in ambiguous select matches ({gh-issue}`2759`) and should be much more efficient + when running `bazel query` due to fewer repositories being included + ({gh-issue}`2849`). {#v0-0-0-added} ### Added From a0b7f437dea15fd26b6454867a3195bbf9b38889 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 8 Jul 2025 18:41:14 +0900 Subject: [PATCH 19/29] fix defaults and add docs --- MODULE.bazel | 12 ++++++------ python/private/pypi/extension.bzl | 24 ++++++++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 765102da90..5b40a1e984 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -75,9 +75,9 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") os_name = "linux", platform = "linux_{}".format(cpu), whl_abi_tags = [ - "cp{major}{minor}", - "abi3", "none", + "abi3", + "cp{major}{minor}", ], whl_platform_tags = [ "linux_*_{}".format(cpu), @@ -103,9 +103,9 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") os_name = "osx", platform = "osx_{}".format(cpu), whl_abi_tags = [ - "cp{major}{minor}", - "abi3", "none", + "abi3", + "cp{major}{minor}", ], whl_platform_tags = [ "macosx_*_{}".format(suffix) @@ -135,9 +135,9 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") os_name = "windows", platform = "windows_{}".format(cpu), whl_abi_tags = [ - "cp{major}{minor}", - "abi3", "none", + "abi3", + "cp{major}{minor}", ], whl_platform_tags = whl_platform_tags, ) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 0257476b4f..c61202ace0 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -447,9 +447,10 @@ def _configure(config, *, platform, os_name, arch_name, config_settings, env = { arch_name = arch_name, config_settings = config_settings, whl_abi_tags = whl_abi_tags or [ - "cp{major}{minor}", - "abi3", + # NOTE @aignas 2025-07-08: the least preferred is the first item in the list "none", + "abi3", + "cp{major}{minor}", ], whl_platform_tags = whl_platform_tags, env = { @@ -884,21 +885,32 @@ If you are defining custom platforms in your project and don't want things to cl "whl_abi_tags": attr.string_list( doc = """\ A list of ABIs to select wheels for. The values can be either strings or include template -parameters like `{0}` which will be replaced with python version parts. e.g. `cp{0}{1}` will -result in `cp313` given the full python version is `3.13.5`. +parameters like `{major}` and `{minor}` which will be replaced with python version parts. e.g. +`cp{major}{minor}` will result in `cp313` given the full python version is `3.13.5`. + +:::{note} +We select a single wheel and the last match will take precedence. +::: +:::{seealso} See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#abi-tag) for more information. +::: """, ), "whl_platform_tags": attr.string_list( doc = """\ A list of `platform_tag` matchers so that we can select the best wheel based on the user -preference. Per platform we will select a single wheel and the last match from this list will -take precedence. +preference. The items in this list can contain a single `*` character that is equivalent to `.*` regex match. +:::{note} +We select a single wheel and the last match will take precedence. +::: + +:::{seealso} See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag) for more information. +::: """, ), } From 6d6276b36e25831324c1818cdf20599af7b12bd4 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 8 Jul 2025 18:41:57 +0900 Subject: [PATCH 20/29] revert an unnecessary diff --- MODULE.bazel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/MODULE.bazel b/MODULE.bazel index 5b40a1e984..9f3305a938 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -245,6 +245,7 @@ dev_pip = use_extension( dev_dependency = True, ) dev_pip.parse( + download_only = True, experimental_index_url = "https://pypi.org/simple", hub_name = "dev_pip", parallel_download = False, @@ -252,12 +253,14 @@ dev_pip.parse( requirements_lock = "//docs:requirements.txt", ) dev_pip.parse( + download_only = True, experimental_index_url = "https://pypi.org/simple", hub_name = "dev_pip", python_version = "3.13", requirements_lock = "//docs:requirements.txt", ) dev_pip.parse( + download_only = True, experimental_index_url = "https://pypi.org/simple", hub_name = "pypiserver", python_version = "3.11", From 29c82d1bb95e7a23c99f29239818005ac33250d7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 8 Jul 2025 18:44:02 +0900 Subject: [PATCH 21/29] add back the failure --- python/private/pypi/evaluate_markers.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index ee8184ac3b..4d6a39a1df 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -45,7 +45,7 @@ def evaluate_markers(*, requirements, platforms): for platform_str in platform_strings: plat = platforms.get(platform_str) if not plat: - continue + fail("Please define platform: '{}'".format(platform_str)) if evaluate(req.marker, env = plat.env): ret.setdefault(req_string, []).append(platform_str) From a8d67eda0f8b87c523f4949c23001377c7b42d79 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 8 Jul 2025 18:45:47 +0900 Subject: [PATCH 22/29] undo an unnecessary change --- python/private/pypi/extension.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index c61202ace0..93f1c986b5 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -317,7 +317,7 @@ def _create_whl_repos( if v != default }) - for src in sorted(whl.srcs, key = lambda x: x.filename): + for src in whl.srcs: repo = _whl_repo( src = src, whl_library_args = whl_library_args, From 3ec5225728616169c566b724734eacbbe0793365 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 8 Jul 2025 18:46:11 +0900 Subject: [PATCH 23/29] move a note --- python/private/pypi/extension.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 93f1c986b5..944cae9b3a 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -323,9 +323,6 @@ def _create_whl_repos( whl_library_args = whl_library_args, download_only = pip_attr.download_only, netrc = pip_attr.netrc, - # NOTE @aignas 2025-07-07: we guard against an edge-case where there - # are more platforms defined than there are wheels for and users - # disallow building from sdist. use_downloader = use_downloader.get( whl.name, get_index_urls != None, # defaults to True if the get_index_urls is defined @@ -336,6 +333,9 @@ def _create_whl_repos( enable_pipstar = config.enable_pipstar, ) if repo == None: + # NOTE @aignas 2025-07-07: we guard against an edge-case where there + # are more platforms defined than there are wheels for and users + # disallow building from sdist. continue repo_name = "{}_{}".format(pip_name, repo.repo_name) From a19bb082504440b804ef09e3193320a7c338fbc8 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 8 Jul 2025 18:51:26 +0900 Subject: [PATCH 24/29] cleanup --- python/private/pypi/extension.bzl | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 944cae9b3a..fa8cdc14fe 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -429,29 +429,30 @@ def _whl_repo( def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, whl_abi_tags, whl_platform_tags, override = False): """Set the value in the config if the value is provided""" config.setdefault("platforms", {}) - if platform and (os_name or arch_name or config_settings or whl_platform_tags or env): - if not override and config.get("platforms", {}).get(platform): + if platform and (os_name or arch_name or config_settings or whl_abi_tags or whl_platform_tags or env): + if not override and config["platforms"].get(platform): return for key in env: if key not in _SUPPORTED_PEP508_KEYS: fail("Unsupported key in the PEP508 environment: {}".format(key)) + # NOTE @aignas 2025-07-08: the least preferred is the first item in the list if whl_platform_tags and "any" not in whl_platform_tags: # the lowest priority one needs to be the first one whl_platform_tags = ["any"] + whl_platform_tags + whl_abi_tags = whl_abi_tags or ["abi3", "cp{major}{minor}"] + if whl_abi_tags and "none" not in whl_abi_tags: + # the lowest priority one needs to be the first one + whl_abi_tags = ["none"] + whl_abi_tags + config["platforms"][platform] = struct( name = platform.replace("-", "_").lower(), os_name = os_name, arch_name = arch_name, config_settings = config_settings, - whl_abi_tags = whl_abi_tags or [ - # NOTE @aignas 2025-07-08: the least preferred is the first item in the list - "none", - "abi3", - "cp{major}{minor}", - ], + whl_abi_tags = whl_abi_tags, whl_platform_tags = whl_platform_tags, env = { # default to this From d36610f59ee708ec850cbd006965498c7183b134 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Tue, 8 Jul 2025 18:58:11 +0900 Subject: [PATCH 25/29] add notes about defaults --- MODULE.bazel | 3 --- python/private/pypi/extension.bzl | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 9f3305a938..2d3434b4f7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -75,7 +75,6 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") os_name = "linux", platform = "linux_{}".format(cpu), whl_abi_tags = [ - "none", "abi3", "cp{major}{minor}", ], @@ -103,7 +102,6 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") os_name = "osx", platform = "osx_{}".format(cpu), whl_abi_tags = [ - "none", "abi3", "cp{major}{minor}", ], @@ -135,7 +133,6 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") os_name = "windows", platform = "windows_{}".format(cpu), whl_abi_tags = [ - "none", "abi3", "cp{major}{minor}", ], diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index fa8cdc14fe..dbecd75c76 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -888,6 +888,7 @@ If you are defining custom platforms in your project and don't want things to cl A list of ABIs to select wheels for. The values can be either strings or include template parameters like `{major}` and `{minor}` which will be replaced with python version parts. e.g. `cp{major}{minor}` will result in `cp313` given the full python version is `3.13.5`. +Will always include `"none"` even if it is not specified. :::{note} We select a single wheel and the last match will take precedence. @@ -902,6 +903,7 @@ See official [docs](https://packaging.python.org/en/latest/specifications/platfo doc = """\ A list of `platform_tag` matchers so that we can select the best wheel based on the user preference. +Will always include `"any"` even if it is not specified. The items in this list can contain a single `*` character that is equivalent to `.*` regex match. From 6a48f8577f1b15bc50be961cf969e40b11638327 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 12 Jul 2025 23:16:18 +0900 Subject: [PATCH 26/29] correctly select by py_tag --- python/private/pypi/select_whl.bzl | 89 ++++++++++++++-------- tests/pypi/select_whl/select_whl_tests.bzl | 66 ++++++++++++++++ 2 files changed, 125 insertions(+), 30 deletions(-) diff --git a/python/private/pypi/select_whl.bzl b/python/private/pypi/select_whl.bzl index 30d803956f..a6a43f3a0f 100644 --- a/python/private/pypi/select_whl.bzl +++ b/python/private/pypi/select_whl.bzl @@ -4,21 +4,55 @@ load("//python/private:version.bzl", "version") load(":parse_whl_name.bzl", "parse_whl_name") load(":python_tag.bzl", "PY_TAG_GENERIC", "python_tag") -def _get_priority(*, tag, values, allow_wildcard = True): +def _get_priority(*, tags, values, allow_wildcard = True): + keys = [] for priority, wp in enumerate(values): - head, sep, tail = wp.partition("*") - if "*" in tail: - fail("only a single '*' can be present in the matcher") - if not allow_wildcard and sep: - fail("'*' is not allowed in the matcher") + for tag in tags.split("."): + head, sep, tail = wp.partition("*") + if "*" in tail: + fail("only a single '*' can be present in the matcher") + if not allow_wildcard and sep: + fail("'*' is not allowed in the matcher") + + if not sep and tag == head: + keys.append(priority) + elif sep and tag.startswith(head) and tag.endswith(tail): + keys.append(priority) + + if not keys: + return None + + return max(keys) + +def _get_py_priority(*, tags, implementation, py_version): + keys = [] + for tag in tags.split("."): + if tag.startswith(PY_TAG_GENERIC): + ver_str = tag[len(PY_TAG_GENERIC):] + elif tag.startswith(implementation): + ver_str = tag[len(implementation):] + else: + continue + + # Add a 0 at the end in case it is a single digit + ver_str = "{}.{}".format(ver_str[0], ver_str[1:] or "0") + + ver = version.parse(ver_str) + if not version.is_compatible(py_version, ver): + continue + + keys.append(( + tag.startswith(implementation), + version.key(ver), + # Prefer shorter py_tags, which will yield more specialized matches, + # like preferring py3 over py2.py3 + -len(tags), + )) - for p in tag.split("."): - if not sep and p == head: - return priority - elif sep and p.startswith(head) and p.endswith(tail): - return priority + if not keys: + return None - return None + return max(keys) def select_whl(*, whls, python_version, platforms, whl_abi_tags, implementation_name = "cpython", limit = 1, logger = None): """Select a whl that is the most suitable for the given platform. @@ -56,26 +90,22 @@ def select_whl(*, whls, python_version, platforms, whl_abi_tags, implementation_ )) continue - if parsed.python_tag == "py2.py3": - min_version = "2" - else: - min_version = parsed.python_tag[len(implementation):] - - if len(min_version) > 1: - min_version = "{}.{}".format(min_version[0], min_version[1:]) - - min_whl_py_version = version.parse(min_version, strict = True) - if not version.is_ge(py_version, min_whl_py_version): + py_priority = _get_py_priority( + tags = parsed.python_tag, + implementation = implementation, + py_version = py_version, + ) + if py_priority == None: if logger: - logger.debug(lambda: "Discarding the wheel because the min version supported based on the wheel ABI tag '{}' ({}) is not compatible with the provided target Python version '{}'".format( - parsed.abi_tag, - min_whl_py_version.string, + logger.debug(lambda: "The py_tag '{}' does not match implementation version: {} {}".format( + parsed.py_tag, + implementation, py_version.string, )) continue abi_priority = _get_priority( - tag = parsed.abi_tag, + tags = parsed.abi_tag, values = whl_abi_tags, allow_wildcard = False, ) @@ -86,8 +116,9 @@ def select_whl(*, whls, python_version, platforms, whl_abi_tags, implementation_ whl_abi_tags, )) continue + platform_priority = _get_priority( - tag = parsed.platform_tag, + tags = parsed.platform_tag, values = platforms, ) if platform_priority == None: @@ -100,10 +131,8 @@ def select_whl(*, whls, python_version, platforms, whl_abi_tags, implementation_ key = ( # Ensure that we chose the highest compatible version - parsed.python_tag.startswith(implementation), + py_priority, platform_priority, - # prefer abi_tags in this order - version.key(min_whl_py_version), abi_priority, ) candidates.setdefault(key, whl) diff --git a/tests/pypi/select_whl/select_whl_tests.bzl b/tests/pypi/select_whl/select_whl_tests.bzl index 4269680ef8..d7700b127b 100644 --- a/tests/pypi/select_whl/select_whl_tests.bzl +++ b/tests/pypi/select_whl/select_whl_tests.bzl @@ -96,6 +96,55 @@ def _select_whl(whls, debug = False, **kwargs): _tests = [] +def _test_not_select_py2(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = [ + "pkg-0.0.1-py2-none-any.whl", + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + ], + platforms = ["any"], + whl_abi_tags = ["none"], + python_version = "3.13", + limit = 2, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + ) + +_tests.append(_test_not_select_py2) + +def _test_select_cp312(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = [ + "pkg-0.0.1-py2-none-any.whl", + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + "pkg-0.0.1-cp39-none-any.whl", + "pkg-0.0.1-cp312-none-any.whl", + "pkg-0.0.1-cp314-none-any.whl", + ], + platforms = ["any"], + whl_abi_tags = ["none"], + python_version = "3.13", + limit = 5, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + "pkg-0.0.1-cp39-none-any.whl", + "pkg-0.0.1-cp312-none-any.whl", + ) + +_tests.append(_test_select_cp312) + def _test_simplest(env): whls = [ "pkg-0.0.1-py2.py3-abi3-any.whl", @@ -281,6 +330,23 @@ def _test_freethreaded_wheels(env): _tests.append(_test_freethreaded_wheels) +def _test_pytags_all_possible(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313-none-win_amd64.whl", + ], + platforms = ["win_amd64"], + whl_abi_tags = ["none"], + python_version = "3.12", + ) + _match( + env, + [got], + "pkg-0.0.1-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313-none-win_amd64.whl", + ) + +_tests.append(_test_pytags_all_possible) + def select_whl_test_suite(name): """Create the test suite. From 281f7fcbce581cfdb2d11667834dfad9904547ef Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 12 Jul 2025 23:50:55 +0900 Subject: [PATCH 27/29] refactor/cleanup code --- python/private/pypi/select_whl.bzl | 163 +++++++++++++++-------------- 1 file changed, 82 insertions(+), 81 deletions(-) diff --git a/python/private/pypi/select_whl.bzl b/python/private/pypi/select_whl.bzl index a6a43f3a0f..557b18f51c 100644 --- a/python/private/pypi/select_whl.bzl +++ b/python/private/pypi/select_whl.bzl @@ -4,55 +4,80 @@ load("//python/private:version.bzl", "version") load(":parse_whl_name.bzl", "parse_whl_name") load(":python_tag.bzl", "PY_TAG_GENERIC", "python_tag") -def _get_priority(*, tags, values, allow_wildcard = True): +def _get_priority(*, tag, values, allow_wildcard = True): keys = [] for priority, wp in enumerate(values): - for tag in tags.split("."): - head, sep, tail = wp.partition("*") - if "*" in tail: - fail("only a single '*' can be present in the matcher") - if not allow_wildcard and sep: - fail("'*' is not allowed in the matcher") - - if not sep and tag == head: - keys.append(priority) - elif sep and tag.startswith(head) and tag.endswith(tail): - keys.append(priority) + head, sep, tail = wp.partition("*") + if "*" in tail: + fail("only a single '*' can be present in the matcher") + if not allow_wildcard and sep: + fail("'*' is not allowed in the matcher") + + if not sep and tag == head: + keys.append(priority) + elif sep and tag.startswith(head) and tag.endswith(tail): + keys.append(priority) if not keys: return None return max(keys) -def _get_py_priority(*, tags, implementation, py_version): - keys = [] - for tag in tags.split("."): - if tag.startswith(PY_TAG_GENERIC): - ver_str = tag[len(PY_TAG_GENERIC):] - elif tag.startswith(implementation): - ver_str = tag[len(implementation):] - else: - continue +def _get_py_priority(*, tag, implementation, py_version): + if tag.startswith(PY_TAG_GENERIC): + ver_str = tag[len(PY_TAG_GENERIC):] + elif tag.startswith(implementation): + ver_str = tag[len(implementation):] + else: + return None - # Add a 0 at the end in case it is a single digit - ver_str = "{}.{}".format(ver_str[0], ver_str[1:] or "0") + # Add a 0 at the end in case it is a single digit + ver_str = "{}.{}".format(ver_str[0], ver_str[1:] or "0") - ver = version.parse(ver_str) - if not version.is_compatible(py_version, ver): - continue + ver = version.parse(ver_str) + if not version.is_compatible(py_version, ver): + return None - keys.append(( - tag.startswith(implementation), - version.key(ver), - # Prefer shorter py_tags, which will yield more specialized matches, - # like preferring py3 over py2.py3 - -len(tags), - )) + return ( + tag.startswith(implementation), + version.key(ver), + ) - if not keys: - return None +def _tag_sets(*, whls, implementation, py_version, whl_abi_tags, platforms): + ret = {} + for whl in whls: + parsed = parse_whl_name(whl.filename) - return max(keys) + # See https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#compressed-tag-sets + for py in parsed.python_tag.split("."): + py = _get_py_priority( + tag = py, + implementation = implementation, + py_version = py_version, + ) + if py == None: + ret[struct(py = py)] = whl + continue + + for abi in parsed.abi_tag.split("."): + abi = _get_priority( + tag = abi, + values = whl_abi_tags, + allow_wildcard = False, + ) + if py == None: + ret[struct(py = py, abi = abi)] = whl + continue + + for p in parsed.platform_tag.split("."): + platform = _get_priority( + tag = p, + values = platforms, + ) + + ret[struct(py = py, abi = abi, platform = platform)] = whl + + return ret def select_whl(*, whls, python_version, platforms, whl_abi_tags, implementation_name = "cpython", limit = 1, logger = None): """Select a whl that is the most suitable for the given platform. @@ -77,65 +102,41 @@ def select_whl(*, whls, python_version, platforms, whl_abi_tags, implementation_ candidates = {} implementation = python_tag(implementation_name) - for whl in whls: - parsed = parse_whl_name(whl.filename) - - if parsed.python_tag.startswith(PY_TAG_GENERIC): - pass - elif not parsed.python_tag.startswith(implementation): + for priority, whl in _tag_sets( + whls = whls, + implementation = implementation, + py_version = py_version, + whl_abi_tags = whl_abi_tags, + platforms = platforms, + ).items(): + if priority.py == None: if logger: - logger.debug(lambda: "Discarding the wheel because the implementation '{}' is not compatible with target implementation '{}'".format( - parsed.python_tag, - implementation, - )) - continue - - py_priority = _get_py_priority( - tags = parsed.python_tag, - implementation = implementation, - py_version = py_version, - ) - if py_priority == None: - if logger: - logger.debug(lambda: "The py_tag '{}' does not match implementation version: {} {}".format( - parsed.py_tag, + logger.debug(lambda: "The python_tag in '{}' does not match implementation or version: {} {}".format( + whl.filename, implementation, py_version.string, )) continue - - abi_priority = _get_priority( - tags = parsed.abi_tag, - values = whl_abi_tags, - allow_wildcard = False, - ) - if abi_priority == None: + elif priority.abi == None: if logger: - logger.debug(lambda: "The abi '{}' does not match given list: {}".format( - parsed.abi_tag, + logger.debug(lambda: "The abi_tag in '{}' does not match given list: {}".format( + whl.filename, whl_abi_tags, )) continue - - platform_priority = _get_priority( - tags = parsed.platform_tag, - values = platforms, - ) - if platform_priority == None: + elif priority.platform == None: if logger: - logger.debug(lambda: "The platform_tag '{}' does not match given list: {}".format( - parsed.platform_tag, + logger.debug(lambda: "The platform_tag in '{}' does not match given list: {}".format( + whl.filename, platforms, )) continue - key = ( - # Ensure that we chose the highest compatible version - py_priority, - platform_priority, - abi_priority, - ) - candidates.setdefault(key, whl) + candidates.setdefault(( + priority.platform, # Prefer platform wheels + priority.py, # Then prefer implementation/python version + priority.abi, # Then prefer more specific ABI wheels + ), whl) if not candidates: return None From a53c1e8cac3fafb7b8367ef7cd1983677f3f04aa Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 13 Jul 2025 22:07:32 +0900 Subject: [PATCH 28/29] cleanup --- python/private/pypi/select_whl.bzl | 126 ++++++++++++--------- tests/pypi/select_whl/select_whl_tests.bzl | 45 ++++++++ 2 files changed, 115 insertions(+), 56 deletions(-) diff --git a/python/private/pypi/select_whl.bzl b/python/private/pypi/select_whl.bzl index 557b18f51c..395b8ade96 100644 --- a/python/private/pypi/select_whl.bzl +++ b/python/private/pypi/select_whl.bzl @@ -4,7 +4,7 @@ load("//python/private:version.bzl", "version") load(":parse_whl_name.bzl", "parse_whl_name") load(":python_tag.bzl", "PY_TAG_GENERIC", "python_tag") -def _get_priority(*, tag, values, allow_wildcard = True): +def _priority_by_values(*, tag, values, allow_wildcard = True): keys = [] for priority, wp in enumerate(values): head, sep, tail = wp.partition("*") @@ -18,12 +18,9 @@ def _get_priority(*, tag, values, allow_wildcard = True): elif sep and tag.startswith(head) and tag.endswith(tail): keys.append(priority) - if not keys: - return None - - return max(keys) + return max(keys) if keys else None -def _get_py_priority(*, tag, implementation, py_version): +def _priority_by_version(*, tag, implementation, py_version): if tag.startswith(PY_TAG_GENERIC): ver_str = tag[len(PY_TAG_GENERIC):] elif tag.startswith(implementation): @@ -43,39 +40,83 @@ def _get_py_priority(*, tag, implementation, py_version): version.key(ver), ) -def _tag_sets(*, whls, implementation, py_version, whl_abi_tags, platforms): +def _candidates_by_priority(*, whls, implementation, py_version, whl_abi_tags, platforms, logger): + """Calculate the priority of each wheel + + Args: + whls: {type}`list[struct]` The whls to select from. + implementation: {type}`str` The target Python implementation. + py_version: {type}`struct` The target python version. + whl_abi_tags: {type}`list[str]` The whl abi tags to select from. + platforms: {type}`list[str]` The whl platform tags to select from. + logger: The logger to use for debugging info + + Returns: + A dictionary where keys are priority tuples which allows us to sort and pick the + last item. + """ + ret = {} for whl in whls: parsed = parse_whl_name(whl.filename) + priority = None # See https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#compressed-tag-sets - for py in parsed.python_tag.split("."): - py = _get_py_priority( - tag = py, - implementation = implementation, - py_version = py_version, - ) - if py == None: - ret[struct(py = py)] = whl + for platform in parsed.platform_tag.split("."): + platform = _priority_by_values(tag = platform, values = platforms) + if platform == None: + if logger: + logger.debug(lambda: "The platform_tag in '{}' does not match given list: {}".format( + whl.filename, + platforms, + )) continue - for abi in parsed.abi_tag.split("."): - abi = _get_priority( - tag = abi, - values = whl_abi_tags, - allow_wildcard = False, + for py in parsed.python_tag.split("."): + py = _priority_by_version( + tag = py, + implementation = implementation, + py_version = py_version, ) if py == None: - ret[struct(py = py, abi = abi)] = whl + if logger: + logger.debug(lambda: "The python_tag in '{}' does not match implementation or version: {} {}".format( + whl.filename, + implementation, + py_version.string, + )) continue - for p in parsed.platform_tag.split("."): - platform = _get_priority( - tag = p, - values = platforms, + for abi in parsed.abi_tag.split("."): + abi = _priority_by_values( + tag = abi, + values = whl_abi_tags, + allow_wildcard = False, ) + if abi == None: + if logger: + logger.debug(lambda: "The abi_tag in '{}' does not match given list: {}".format( + whl.filename, + whl_abi_tags, + )) + continue + + # 1. Prefer platform wheels + # 2. Then prefer implementation/python version + # 3. Then prefer more specific ABI wheels + candidate = (platform, py, abi) + priority = priority or candidate + if candidate > priority: + priority = candidate + + if priority == None: + if logger: + logger.debug(lambda: "The whl '{}' is incompatible".format( + whl.filename, + )) + continue - ret[struct(py = py, abi = abi, platform = platform)] = whl + ret[priority] = whl return ret @@ -102,41 +143,14 @@ def select_whl(*, whls, python_version, platforms, whl_abi_tags, implementation_ candidates = {} implementation = python_tag(implementation_name) - for priority, whl in _tag_sets( + candidates = _candidates_by_priority( whls = whls, implementation = implementation, py_version = py_version, whl_abi_tags = whl_abi_tags, platforms = platforms, - ).items(): - if priority.py == None: - if logger: - logger.debug(lambda: "The python_tag in '{}' does not match implementation or version: {} {}".format( - whl.filename, - implementation, - py_version.string, - )) - continue - elif priority.abi == None: - if logger: - logger.debug(lambda: "The abi_tag in '{}' does not match given list: {}".format( - whl.filename, - whl_abi_tags, - )) - continue - elif priority.platform == None: - if logger: - logger.debug(lambda: "The platform_tag in '{}' does not match given list: {}".format( - whl.filename, - platforms, - )) - continue - - candidates.setdefault(( - priority.platform, # Prefer platform wheels - priority.py, # Then prefer implementation/python version - priority.abi, # Then prefer more specific ABI wheels - ), whl) + logger = logger, + ) if not candidates: return None diff --git a/tests/pypi/select_whl/select_whl_tests.bzl b/tests/pypi/select_whl/select_whl_tests.bzl index d7700b127b..06da2ab8ae 100644 --- a/tests/pypi/select_whl/select_whl_tests.bzl +++ b/tests/pypi/select_whl/select_whl_tests.bzl @@ -118,6 +118,29 @@ def _test_not_select_py2(env): _tests.append(_test_not_select_py2) +def _test_not_select_abi3(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-any.whl", + # the following should be ignored + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-p1.p2.p2.whl", + ], + platforms = ["any", "p1"], + whl_abi_tags = ["none"], + python_version = "3.13", + limit = 2, + debug = True, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + ) + +_tests.append(_test_not_select_abi3) + def _test_select_cp312(env): # Check we prefer platform specific wheels got = _select_whl( @@ -347,6 +370,28 @@ def _test_pytags_all_possible(env): _tests.append(_test_pytags_all_possible) +def _test_manylinx_musllinux_pref(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-manylinux_2_31_x86_64.musllinux_1_1_x86_64.whl", + ], + platforms = [ + "manylinux_*_x86_64", + "musllinux_*_x86_64", + ], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, + ) + _match( + env, + got, + # there is only wheel, just select that + "pkg-0.0.1-py3-none-manylinux_2_31_x86_64.musllinux_1_1_x86_64.whl", + ) + +_tests.append(_test_manylinx_musllinux_pref) + def select_whl_test_suite(name): """Create the test suite. From 82e153251aac435b35ddb0f013dff6418036ea1f Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sun, 13 Jul 2025 22:19:11 +0900 Subject: [PATCH 29/29] add a note --- python/private/pypi/extension.bzl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index dbecd75c76..b8023a2d1c 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -911,6 +911,16 @@ The items in this list can contain a single `*` character that is equivalent to We select a single wheel and the last match will take precedence. ::: +:::{note} +The following tag prefixes should be used instead of the legacy equivalents: +* `manylinux_2_5` instead of `manylinux1` +* `manylinux_2_12` instead of `manylinux2010` +* `manylinux_2_17` instead of `manylinux2014` + +When parsing the whl filenames `rules_python` will automatically transform wheel filenames to the +latest format. +::: + :::{seealso} See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag) for more information. :::