diff --git a/.bazelrc b/.bazelrc index d7e1771336..8997db9f91 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,rules_python-repro,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,gazelle/python/private,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered,tests/modules/another_module,tests/modules/other,tests/modules/other/nspkg_delta,tests/modules/other/nspkg_gamma,tests/modules/other/nspkg_single,tests/modules/other/simple_v1,tests/modules/other/simple_v2,tests/modules/other/with_external_data,tests/whl_with_build_files/testdata,tests/whl_with_build_files/testdata/somepkg,tests/whl_with_build_files/testdata/somepkg-1.0.dist-info,tests/whl_with_build_files/testdata/somepkg/subpkg test --test_output=errors diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f02c8bbb4..a8e0124742 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,10 @@ END_UNRELEASED_TEMPLATE * (toolchain) Python 3.13 now references 3.13.5 * (gazelle) Switched back to smacker/go-tree-sitter, fixing [#2630](https://github.com/bazel-contrib/rules_python/issues/2630) +* (pypi) From now on the list of default platforms only includes `linux_x86_64`, `linux_aarch64`, + `osx_x86_64`, `osx_aarch64` and `windows_x86_64`. If you are on other platforms, you need to + use the `pip.default` to configure it yourself. If you are interested in graduating the + platform, consider helping set us up CI for them and update the documentation. {#v0-0-0-fixed} ### Fixed @@ -86,6 +90,10 @@ END_UNRELEASED_TEMPLATE ({gh-issue}`3043`). * (pypi) The pipstar `defaults` configuration now supports any custom platform name. +* (pypi) The selection of the whls has been changed and should no longer result + in ambiguous select matches ({gh-issue}`2759`) and should be much more efficient + when running `bazel query` due to fewer repositories being included + ({gh-issue}`2849`). {#v0-0-0-added} ### Added diff --git a/MODULE.bazel b/MODULE.bazel index 9db287dc28..2d3434b4f7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -74,16 +74,18 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") env = {"platform_version": "0"}, os_name = "linux", platform = "linux_{}".format(cpu), + whl_abi_tags = [ + "abi3", + "cp{major}{minor}", + ], + whl_platform_tags = [ + "linux_*_{}".format(cpu), + "manylinux_*_{}".format(cpu), + ], ) for cpu in [ "x86_64", "aarch64", - # TODO @aignas 2025-05-19: only leave tier 0-1 cpus when stabilizing the - # `pip.default` extension. i.e. drop the below values - users will have to - # define themselves if they need them. - "arm", - "ppc", - "s390x", ] ] @@ -99,26 +101,53 @@ pip = use_extension("//python/extensions:pip.bzl", "pip") env = {"platform_version": "14.0"}, os_name = "osx", platform = "osx_{}".format(cpu), + whl_abi_tags = [ + "abi3", + "cp{major}{minor}", + ], + whl_platform_tags = [ + "macosx_*_{}".format(suffix) + for suffix in platform_tag_cpus + ], ) - for cpu in [ - "aarch64", - "x86_64", - ] + for cpu, platform_tag_cpus in { + "aarch64": [ + "universal2", + "arm64", + ], + "x86_64": [ + "universal2", + "x86_64", + ], + }.items() +] + +[ + pip.default( + arch_name = cpu, + config_settings = [ + "@platforms//cpu:{}".format(cpu), + "@platforms//os:windows", + ], + env = {"platform_version": "0"}, + os_name = "windows", + platform = "windows_{}".format(cpu), + whl_abi_tags = [ + "abi3", + "cp{major}{minor}", + ], + whl_platform_tags = whl_platform_tags, + ) + for cpu, whl_platform_tags in { + "x86_64": ["win_amd64"], + }.items() ] -pip.default( - arch_name = "x86_64", - config_settings = [ - "@platforms//cpu:x86_64", - "@platforms//os:windows", - ], - env = {"platform_version": "0"}, - os_name = "windows", - platform = "windows_x86_64", -) pip.parse( # NOTE @aignas 2024-10-26: We have an integration test that depends on us # being able to build sdists for this hub, so explicitly set this to False. + # + # how do we test sdists? Maybe just worth adding a single sdist somewhere? download_only = False, experimental_index_url = "https://pypi.org/simple", hub_name = "rules_python_publish_deps", diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 841c096dcf..95e1090f53 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -158,6 +158,23 @@ pip.whl_mods( ) use_repo(pip, "whl_mods_hub") +# Because below we are using `windows_aarch64` platform, we have to define various +# properties for it. +pip.default( + arch_name = "aarch64", + config_settings = [ + "@platforms//os:windows", + "@platforms//cpu:aarch64", + ], + env = { + "platform_version": "0", + }, + os_name = "windows", + platform = "windows_aarch64", + whl_abi_tags = [], # default to all ABIs + whl_platform_tags = ["win_amd64"], +) + # To fetch pip dependencies, use pip.parse. We can pass in various options, # but typically we pass requirements and the Python version. The Python # version must have been configured by a corresponding `python.toolchain()` diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index b098f29e94..71998e0620 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -62,8 +62,6 @@ bzl_library( name = "config_settings_bzl", srcs = ["config_settings.bzl"], deps = [ - ":flags_bzl", - "//python/private:flags_bzl", "@bazel_skylib//lib:selects", ], ) @@ -116,11 +114,11 @@ bzl_library( ":parse_whl_name_bzl", ":pep508_env_bzl", ":pip_repository_attrs_bzl", + ":python_tag_bzl", ":simpleapi_download_bzl", ":whl_config_setting_bzl", ":whl_library_bzl", ":whl_repo_name_bzl", - ":whl_target_platforms_bzl", "//python/private:full_version_bzl", "//python/private:normalize_name_bzl", "//python/private:version_bzl", @@ -209,7 +207,7 @@ bzl_library( ":parse_requirements_txt_bzl", ":pypi_repo_utils_bzl", ":requirements_files_by_platform_bzl", - ":whl_target_platforms_bzl", + ":select_whl_bzl", "//python/private:normalize_name_bzl", "//python/private:repo_utils_bzl", ], @@ -263,11 +261,6 @@ bzl_library( ], ) -bzl_library( - name = "pep508_platform_bzl", - srcs = ["pep508_platform.bzl"], -) - bzl_library( name = "pep508_requirement_bzl", srcs = ["pep508_requirement.bzl"], @@ -322,8 +315,6 @@ bzl_library( srcs = ["pkg_aliases.bzl"], deps = [ ":labels_bzl", - ":parse_whl_name_bzl", - ":whl_target_platforms_bzl", "//python/private:text_util_bzl", "@bazel_skylib//lib:selects", ], @@ -338,14 +329,20 @@ bzl_library( ], ) +bzl_library( + name = "python_tag_bzl", + srcs = ["python_tag.bzl"], + deps = [ + "//python/private:version_bzl", + ], +) + bzl_library( name = "render_pkg_aliases_bzl", srcs = ["render_pkg_aliases.bzl"], deps = [ ":generate_group_library_build_bazel_bzl", - ":parse_whl_name_bzl", ":whl_config_setting_bzl", - ":whl_target_platforms_bzl", "//python/private:normalize_name_bzl", "//python/private:text_util_bzl", ], @@ -359,6 +356,16 @@ bzl_library( ], ) +bzl_library( + name = "select_whl_bzl", + srcs = ["select_whl.bzl"], + deps = [ + ":parse_whl_name_bzl", + ":python_tag_bzl", + "//python/private:version_bzl", + ], +) + bzl_library( name = "simpleapi_download_bzl", srcs = ["simpleapi_download.bzl"], @@ -422,5 +429,4 @@ bzl_library( bzl_library( name = "whl_target_platforms_bzl", srcs = ["whl_target_platforms.bzl"], - deps = [":parse_whl_name_bzl"], ) diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl index f4826007f8..47de287071 100644 --- a/python/private/pypi/config_settings.bzl +++ b/python/private/pypi/config_settings.bzl @@ -16,102 +16,13 @@ The {obj}`config_settings` macro is used to create the config setting targets that can be used in the {obj}`pkg_aliases` macro for selecting the compatible repositories. - -Bazel's selects work by selecting the most-specialized configuration setting -that matches the target platform, which is further described in [bazel documentation][docs]. -We can leverage this fact to ensure that the most specialized matches are used -by default with the users being able to configure string_flag values to select -the less specialized ones. - -[docs]: https://bazel.build/docs/configurable-attributes - -The config settings in the order from the least specialized to the most -specialized is as follows: -* `:is_cp3` -* `:is_cp3_sdist` -* `:is_cp3_py_none_any` -* `:is_cp3_py3_none_any` -* `:is_cp3_py3_abi3_any` -* `:is_cp3_none_any` -* `:is_cp3_any_any` -* `:is_cp3_cp3_any` and `:is_cp3_cp3t_any` -* `:is_cp3_py_none_` -* `:is_cp3_py3_none_` -* `:is_cp3_py3_abi3_` -* `:is_cp3_none_` -* `:is_cp3_abi3_` -* `:is_cp3_cp3_` and `:is_cp3_cp3t_` - -Optionally instead of `` there sometimes may be `.` used in order to fully specify the versions - -The specialization of free-threaded vs non-free-threaded wheels is the same as -they are just variants of each other. The same goes for the specialization of -`musllinux` vs `manylinux`. - -The goal of this macro is to provide config settings that provide unambigous -matches if any pair of them is used together for any target configuration -setting. We achieve this by using dummy internal `flag_values` keys to force the -items further down the list to appear to be more specialized than the ones above. - -What is more, the names of the config settings are as similar to the platform wheel -specification as possible. How the wheel names map to the config setting names defined -in here is described in {obj}`pkg_aliases` documentation. - -:::{note} -Right now the specialization of adjacent config settings where one is with -`constraint_values` and one is without is ambiguous. I.e. `py_none_any` and -`sdist_linux_x86_64` have the same specialization from bazel point of view -because one has one `flag_value` entry and `constraint_values` and the -other has 2 flag_value entries. And unfortunately there is no way to disambiguate -it, because we are essentially in two dimensions here (`flag_values` and -`constraint_values`). Hence, when using the `config_settings` from here, -either have all of them with empty `suffix` or all of them with a non-empty -suffix. -::: """ load("@bazel_skylib//lib:selects.bzl", "selects") -load("//python/private:flags.bzl", "LibcFlag") -load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag") - -FLAGS = struct( - **{ - f: str(Label("//python/config_settings:" + f)) - for f in [ - "is_pip_whl_auto", - "is_pip_whl_no", - "is_pip_whl_only", - "_is_py_freethreaded_yes", - "_is_py_freethreaded_no", - "pip_whl_glibc_version", - "pip_whl_muslc_version", - "pip_whl_osx_arch", - "pip_whl_osx_version", - "py_linux_libc", - "python_version", - ] - } -) - -_DEFAULT = "//conditions:default" -_INCOMPATIBLE = "@platforms//:incompatible" - -# Here we create extra string flags that are just to work with the select -# selecting the most specialized match. We don't allow the user to change -# them. -_flags = struct( - **{ - f: str(Label("//python/config_settings:_internal_pip_" + f)) - for f in INTERNAL_FLAGS - } -) def config_settings( *, python_versions = [], - glibc_versions = [], - muslc_versions = [], - osx_versions = [], name = None, platform_config_settings = {}, **kwargs): @@ -121,12 +32,6 @@ def config_settings( name (str): Currently unused. python_versions (list[str]): The list of python versions to configure config settings for. - glibc_versions (list[str]): The list of glibc version of the wheels to - configure config settings for. - muslc_versions (list[str]): The list of musl version of the wheels to - configure config settings for. - osx_versions (list[str]): The list of OSX OS versions to configure - config settings for. platform_config_settings: {type}`dict[str, list[str]]` the constraint values to use instead of the default ones. Key are platform names (a human-friendly platform string). Values are lists of @@ -135,48 +40,22 @@ def config_settings( {obj}`native`. """ - glibc_versions = [""] + glibc_versions - muslc_versions = [""] + muslc_versions - osx_versions = [""] + osx_versions target_platforms = { "": [], - # TODO @aignas 2025-06-15: allowing universal2 and platform specific wheels in one - # closure is making things maybe a little bit too complicated. - "osx_universal2": ["@platforms//os:osx"], } | platform_config_settings for python_version in python_versions: for platform_name, config_settings in target_platforms.items(): suffix = "_{}".format(platform_name) if platform_name else "" - os, _, cpu = platform_name.partition("_") - - # We parse the target settings and if there is a "platforms//os" or - # "platforms//cpu" value in here, we also add it into the constraint_values - # - # this is to ensure that we can still pass all of the unit tests for config - # setting specialization. - constraint_values = [] - for setting in config_settings: - setting_label = Label(setting) - if setting_label.repo_name == "platforms" and setting_label.package in ["os", "cpu"]: - constraint_values.append(setting) _dist_config_settings( suffix = suffix, - plat_flag_values = _plat_flag_values( - os = os, - cpu = cpu, - osx_versions = osx_versions, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - ), config_settings = config_settings, - constraint_values = constraint_values, python_version = python_version, **kwargs ) -def _dist_config_settings(*, suffix, plat_flag_values, python_version, **kwargs): +def _dist_config_settings(*, suffix, python_version, **kwargs): flag_values = { Label("//python/config_settings:python_version_major_minor"): python_version, } @@ -190,156 +69,11 @@ def _dist_config_settings(*, suffix, plat_flag_values, python_version, **kwargs) **kwargs ) - flag_values[_flags.dist] = "" - - # First create an sdist, we will be building upon the flag values, which - # will ensure that each sdist config setting is the least specialized of - # all. However, we need at least one flag value to cover the case where we - # have `sdist` for any platform, hence we have a non-empty `flag_values` - # here. - _dist_config_setting( - name = "{}_sdist{}".format(prefix, suffix), - flag_values = flag_values, - compatible_with = (FLAGS.is_pip_whl_no, FLAGS.is_pip_whl_auto), - **kwargs - ) - - used_flags = {} - - # NOTE @aignas 2024-12-01: the abi3 is not compatible with freethreaded - # builds as per PEP703 (https://peps.python.org/pep-0703/#backwards-compatibility) - # - # The discussion here also reinforces this notion: - # https://discuss.python.org/t/pep-703-making-the-global-interpreter-lock-optional-3-12-updates/26503/99 - - for name, f, compatible_with in [ - ("py_none", _flags.whl, None), - ("py3_none", _flags.whl_py3, None), - ("py3_abi3", _flags.whl_py3_abi3, (FLAGS._is_py_freethreaded_no,)), - ("none", _flags.whl_pycp3x, None), - ("abi3", _flags.whl_pycp3x_abi3, (FLAGS._is_py_freethreaded_no,)), - # The below are not specializations of one another, they are variants - (cpv, _flags.whl_pycp3x_abicp, (FLAGS._is_py_freethreaded_no,)), - (cpv + "t", _flags.whl_pycp3x_abicp, (FLAGS._is_py_freethreaded_yes,)), - ]: - if (f, compatible_with) in used_flags: - # This should never happen as all of the different whls should have - # unique flag values - fail("BUG: the flag {} is attempted to be added twice to the list".format(f)) - else: - flag_values[f] = "yes" if f == _flags.whl else "" - used_flags[(f, compatible_with)] = True - - _dist_config_setting( - name = "{}_{}_any{}".format(prefix, name, suffix), - flag_values = flag_values, - compatible_with = compatible_with, - **kwargs - ) - - generic_flag_values = flag_values - generic_used_flags = used_flags - - for (suffix, flag_values) in plat_flag_values: - used_flags = {(f, None): True for f in flag_values} | generic_used_flags - flag_values = flag_values | generic_flag_values - - for name, f, compatible_with in [ - ("py_none", _flags.whl_plat, None), - ("py3_none", _flags.whl_plat_py3, None), - ("py3_abi3", _flags.whl_plat_py3_abi3, (FLAGS._is_py_freethreaded_no,)), - ("none", _flags.whl_plat_pycp3x, None), - ("abi3", _flags.whl_plat_pycp3x_abi3, (FLAGS._is_py_freethreaded_no,)), - # The below are not specializations of one another, they are variants - (cpv, _flags.whl_plat_pycp3x_abicp, (FLAGS._is_py_freethreaded_no,)), - (cpv + "t", _flags.whl_plat_pycp3x_abicp, (FLAGS._is_py_freethreaded_yes,)), - ]: - if (f, compatible_with) in used_flags: - # This should never happen as all of the different whls should have - # unique flag values. - fail("BUG: the flag {} is attempted to be added twice to the list".format(f)) - else: - flag_values[f] = "" - used_flags[(f, compatible_with)] = True - - _dist_config_setting( - name = "{}_{}_{}".format(prefix, name, suffix), - flag_values = flag_values, - compatible_with = compatible_with, - **kwargs - ) - -def _to_version_string(version, sep = "."): - if not version: - return "" - - return "{}{}{}".format(version[0], sep, version[1]) - -def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions): - ret = [] - if os == "": - return [] - elif os == "windows": - ret.append(("{}_{}".format(os, cpu), {})) - elif os == "osx": - for osx_version in osx_versions: - flags = { - FLAGS.pip_whl_osx_version: _to_version_string(osx_version), - } - if cpu != "universal2": - flags[FLAGS.pip_whl_osx_arch] = UniversalWhlFlag.ARCH - - if not osx_version: - suffix = "{}_{}".format(os, cpu) - else: - suffix = "{}_{}_{}".format(os, _to_version_string(osx_version, "_"), cpu) - - ret.append((suffix, flags)) - - elif os == "linux": - for os_prefix, linux_libc in { - os: LibcFlag.GLIBC, - "many" + os: LibcFlag.GLIBC, - "musl" + os: LibcFlag.MUSL, - }.items(): - if linux_libc == LibcFlag.GLIBC: - libc_versions = glibc_versions - libc_flag = FLAGS.pip_whl_glibc_version - elif linux_libc == LibcFlag.MUSL: - libc_versions = muslc_versions - libc_flag = FLAGS.pip_whl_muslc_version - else: - fail("Unsupported libc type: {}".format(linux_libc)) - - for libc_version in libc_versions: - if libc_version and os_prefix == os: - continue - elif libc_version: - suffix = "{}_{}_{}".format(os_prefix, _to_version_string(libc_version, "_"), cpu) - else: - suffix = "{}_{}".format(os_prefix, cpu) - - ret.append(( - suffix, - { - FLAGS.py_linux_libc: linux_libc, - libc_flag: _to_version_string(libc_version), - }, - )) - else: - fail("Unsupported os: {}".format(os)) - - return ret - -def _dist_config_setting(*, name, compatible_with = None, selects = selects, native = native, config_settings = None, **kwargs): +def _dist_config_setting(*, name, selects = selects, native = native, config_settings = None, **kwargs): """A macro to create a target for matching Python binary and source distributions. Args: name: The name of the public target. - compatible_with: {type}`tuple[Label]` A collection of config settings that are - compatible with the given dist config setting. For example, if only - non-freethreaded python builds are allowed, add - FLAGS._is_py_freethreaded_no here. config_settings: {type}`list[str | Label]` the list of target settings that must be matched before we try to evaluate the config_setting that we may create in this function. @@ -352,18 +86,6 @@ def _dist_config_setting(*, name, compatible_with = None, selects = selects, nat **kwargs: The kwargs passed to the config_setting rule. Visibility of the main alias target is also taken from the kwargs. """ - if compatible_with: - dist_config_setting_name = "_" + name - native.alias( - name = name, - actual = select( - {setting: dist_config_setting_name for setting in compatible_with} | { - _DEFAULT: _INCOMPATIBLE, - }, - ), - visibility = kwargs.get("visibility"), - ) - name = dist_config_setting_name # first define the config setting that has all of the constraint values _name = "_" + name diff --git a/python/private/pypi/evaluate_markers.bzl b/python/private/pypi/evaluate_markers.bzl index 6167cdbc96..4d6a39a1df 100644 --- a/python/private/pypi/evaluate_markers.bzl +++ b/python/private/pypi/evaluate_markers.bzl @@ -43,11 +43,11 @@ def evaluate_markers(*, requirements, platforms): for req_string, platform_strings in requirements.items(): req = requirement(req_string) for platform_str in platform_strings: - env = platforms.get(platform_str) - if not env: + plat = platforms.get(platform_str) + if not plat: fail("Please define platform: '{}'".format(platform_str)) - if evaluate(req.marker, env = env): + if evaluate(req.marker, env = plat.env): ret.setdefault(req_string, []).append(platform_str) return ret diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 2c1528e18d..b8023a2d1c 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -31,6 +31,7 @@ load(":parse_requirements.bzl", "parse_requirements") load(":parse_whl_name.bzl", "parse_whl_name") load(":pep508_env.bzl", "env") load(":pip_repository_attrs.bzl", "ATTRS") +load(":python_tag.bzl", "python_tag") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":simpleapi_download.bzl", "simpleapi_download") load(":whl_config_setting.bzl", "whl_config_setting") @@ -68,19 +69,41 @@ def _whl_mods_impl(whl_mods_dict): def _platforms(*, python_version, minor_mapping, config): platforms = {} - python_version = full_version( - version = python_version, - minor_mapping = minor_mapping, + python_version = version.parse( + full_version( + version = python_version, + minor_mapping = minor_mapping, + ), + strict = True, ) - abi = "cp3{}".format(python_version[2:]) for platform, values in config.platforms.items(): - key = "{}_{}".format(abi, platform) - platforms[key] = env(struct( - abi = abi, - os = values.os_name, - arch = values.arch_name, - )) | values.env + # TODO @aignas 2025-07-07: this is probably doing the parsing of the version too + # many times. + key = "{}{}{}.{}_{}".format( + python_tag(values.env["implementation_name"]), + python_version.release[0], + python_version.release[1], + python_version.release[2], + platform, + ) + + platforms[key] = struct( + env = env( + env = values.env, + os = values.os_name, + arch = values.arch_name, + python_version = python_version.string, + ), + whl_abi_tags = [ + v.format( + major = python_version.release[0], + minor = python_version.release[1], + ) + for v in values.whl_abi_tags + ], + whl_platform_tags = values.whl_platform_tags, + ) return platforms def _create_whl_repos( @@ -152,6 +175,8 @@ def _create_whl_repos( )) python_interpreter_target = available_interpreters[python_name] + # TODO @aignas 2025-06-29: we should not need the version in the pip_name if + # we are using pipstar and we are downloading the wheel using the downloader pip_name = "{}_{}".format( hub_name, version_label(pip_attr.python_version), @@ -230,12 +255,21 @@ def _create_whl_repos( ), logger = logger, ), + platforms = _platforms( + python_version = pip_attr.python_version, + minor_mapping = minor_mapping, + config = config, + ), extra_pip_args = pip_attr.extra_pip_args, get_index_urls = get_index_urls, evaluate_markers = evaluate_markers, logger = logger, ) + use_downloader = { + normalize_name(s): False + for s in pip_attr.simpleapi_skip + } exposed_packages = {} for whl in requirements_by_platform: if whl.is_exposed: @@ -289,11 +323,20 @@ def _create_whl_repos( whl_library_args = whl_library_args, download_only = pip_attr.download_only, netrc = pip_attr.netrc, + use_downloader = use_downloader.get( + whl.name, + get_index_urls != None, # defaults to True if the get_index_urls is defined + ), auth_patterns = pip_attr.auth_patterns, python_version = major_minor, is_multiple_versions = whl.is_multiple_versions, enable_pipstar = config.enable_pipstar, ) + if repo == None: + # NOTE @aignas 2025-07-07: we guard against an edge-case where there + # are more platforms defined than there are wheels for and users + # disallow building from sdist. + continue repo_name = "{}_{}".format(pip_name, repo.repo_name) if repo_name in whl_libraries: @@ -312,7 +355,17 @@ def _create_whl_repos( whl_libraries = whl_libraries, ) -def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, netrc, auth_patterns, python_version, enable_pipstar = False): +def _whl_repo( + *, + src, + whl_library_args, + is_multiple_versions, + download_only, + netrc, + auth_patterns, + python_version, + use_downloader, + enable_pipstar = False): args = dict(whl_library_args) args["requirement"] = src.requirement_line is_whl = src.filename.endswith(".whl") @@ -325,19 +378,24 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net args["extra_pip_args"] = src.extra_pip_args if not src.url or (not is_whl and download_only): - # Fallback to a pip-installed wheel - target_platforms = src.target_platforms if is_multiple_versions else [] - return struct( - repo_name = pypi_repo_name( - normalize_name(src.distribution), - *target_platforms - ), - args = args, - config_setting = whl_config_setting( - version = python_version, - target_platforms = target_platforms or None, - ), - ) + if download_only and use_downloader: + # If the user did not allow using sdists and we are using the downloader + # and we are not using simpleapi_skip for this + return None + else: + # Fallback to a pip-installed wheel + target_platforms = src.target_platforms if is_multiple_versions else [] + return struct( + repo_name = pypi_repo_name( + normalize_name(src.distribution), + *target_platforms + ), + args = args, + config_setting = whl_config_setting( + version = python_version, + target_platforms = target_platforms or None, + ), + ) # This is no-op because pip is not used to download the wheel. args.pop("download_only", None) @@ -359,40 +417,47 @@ def _whl_repo(*, src, whl_library_args, is_multiple_versions, download_only, net for p in src.target_platforms ] - # Pure python wheels or sdists may need to have a platform here - target_platforms = None - if is_whl and not src.filename.endswith("-any.whl"): - pass - elif is_multiple_versions: - target_platforms = src.target_platforms - return struct( repo_name = whl_repo_name(src.filename, src.sha256), args = args, config_setting = whl_config_setting( version = python_version, - filename = src.filename, - target_platforms = target_platforms, + target_platforms = src.target_platforms, ), ) -def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, override = False): +def _configure(config, *, platform, os_name, arch_name, config_settings, env = {}, whl_abi_tags, whl_platform_tags, override = False): """Set the value in the config if the value is provided""" config.setdefault("platforms", {}) - if platform: - if not override and config.get("platforms", {}).get(platform): + if platform and (os_name or arch_name or config_settings or whl_abi_tags or whl_platform_tags or env): + if not override and config["platforms"].get(platform): return for key in env: if key not in _SUPPORTED_PEP508_KEYS: fail("Unsupported key in the PEP508 environment: {}".format(key)) + # NOTE @aignas 2025-07-08: the least preferred is the first item in the list + if whl_platform_tags and "any" not in whl_platform_tags: + # the lowest priority one needs to be the first one + whl_platform_tags = ["any"] + whl_platform_tags + + whl_abi_tags = whl_abi_tags or ["abi3", "cp{major}{minor}"] + if whl_abi_tags and "none" not in whl_abi_tags: + # the lowest priority one needs to be the first one + whl_abi_tags = ["none"] + whl_abi_tags + config["platforms"][platform] = struct( name = platform.replace("-", "_").lower(), os_name = os_name, arch_name = arch_name, config_settings = config_settings, - env = env, + whl_abi_tags = whl_abi_tags, + whl_platform_tags = whl_platform_tags, + env = { + # default to this + "implementation_name": "cpython", + } | env, ) else: config["platforms"].pop(platform) @@ -463,14 +528,15 @@ You cannot use both the additive_build_content and additive_build_content_file a env = tag.env, os_name = tag.os_name, platform = tag.platform, + whl_abi_tags = tag.whl_abi_tags, + whl_platform_tags = tag.whl_platform_tags, override = mod.is_root, # TODO @aignas 2025-05-19: add more attr groups: # * for AUTH - the default `netrc` usage could be configured through a common # attribute. # * for index/downloader config. This includes all of those attributes for # overrides, etc. Index overrides per platform could be also used here. - # * for whl selection - selecting preferences of which `platform_tag`s we should use - # for what. We could also model the `cp313t` freethreaded as separate platforms. + # * for whl selection - We could also model the `cp313t` freethreaded as separate platforms. ) config = struct(**defaults) @@ -602,7 +668,14 @@ You cannot use both the additive_build_content and additive_build_content_file a for whl_name, aliases in out.extra_aliases.items(): extra_aliases[hub_name].setdefault(whl_name, {}).update(aliases) exposed_packages.setdefault(hub_name, {}).update(out.exposed_packages) - whl_libraries.update(out.whl_libraries) + for whl_name, lib in out.whl_libraries.items(): + if enable_pipstar: + whl_libraries.setdefault(whl_name, lib) + elif whl_name in lib: + fail("'{}' already in created".format(whl_name)) + else: + # replicate whl_libraries.update(out.whl_libraries) + whl_libraries[whl_name] = lib # TODO @aignas 2024-04-05: how do we support different requirement # cycles for different abis/oses? For now we will need the users to @@ -752,6 +825,7 @@ _default_attrs = { "arch_name": attr.string( doc = """\ The CPU architecture name to be used. +You can use any cpu name from the `@platforms//cpu:` package. :::{note} Either this or {attr}`env` `platform_machine` key should be specified. @@ -765,25 +839,6 @@ The list of labels to `config_setting` targets that need to be matched for the p selected. """, ), - "os_name": attr.string( - doc = """\ -The OS name to be used. - -:::{note} -Either this or the appropriate `env` keys should be specified. -::: -""", - ), - "platform": attr.string( - doc = """\ -A platform identifier which will be used as the unique identifier within the extension evaluation. -If you are defining custom platforms in your project and don't want things to clash, use extension -[isolation] feature. - -[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate -""", - ), -} | { "env": attr.string_dict( doc = """\ The values to use for environment markers when evaluating an expression. @@ -809,6 +864,68 @@ This is only used if the {envvar}`RULES_PYTHON_ENABLE_PIPSTAR` is enabled. """, ), # The values for PEP508 env marker evaluation during the lock file parsing + "os_name": attr.string( + doc = """\ +The OS name to be used. +You can use any OS name from the `@platforms//os:` package. + +:::{note} +Either this or the appropriate `env` keys should be specified. +::: +""", + ), + "platform": attr.string( + doc = """\ +A platform identifier which will be used as the unique identifier within the extension evaluation. +If you are defining custom platforms in your project and don't want things to clash, use extension +[isolation] feature. + +[isolation]: https://bazel.build/rules/lib/globals/module#use_extension.isolate +""", + ), + "whl_abi_tags": attr.string_list( + doc = """\ +A list of ABIs to select wheels for. The values can be either strings or include template +parameters like `{major}` and `{minor}` which will be replaced with python version parts. e.g. +`cp{major}{minor}` will result in `cp313` given the full python version is `3.13.5`. +Will always include `"none"` even if it is not specified. + +:::{note} +We select a single wheel and the last match will take precedence. +::: + +:::{seealso} +See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#abi-tag) for more information. +::: +""", + ), + "whl_platform_tags": attr.string_list( + doc = """\ +A list of `platform_tag` matchers so that we can select the best wheel based on the user +preference. +Will always include `"any"` even if it is not specified. + +The items in this list can contain a single `*` character that is equivalent to `.*` regex match. + +:::{note} +We select a single wheel and the last match will take precedence. +::: + +:::{note} +The following tag prefixes should be used instead of the legacy equivalents: +* `manylinux_2_5` instead of `manylinux1` +* `manylinux_2_12` instead of `manylinux2010` +* `manylinux_2_17` instead of `manylinux2014` + +When parsing the whl filenames `rules_python` will automatically transform wheel filenames to the +latest format. +::: + +:::{seealso} +See official [docs](https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#platform-tag) for more information. +::: +""", + ), } _SUPPORTED_PEP508_KEYS = [ diff --git a/python/private/pypi/flags.bzl b/python/private/pypi/flags.bzl index 037383910e..8aa5c94dd3 100644 --- a/python/private/pypi/flags.bzl +++ b/python/private/pypi/flags.bzl @@ -18,7 +18,7 @@ NOTE: The transitive loads of this should be kept minimal. This avoids loading unnecessary files when all that are needed are flag definitions. """ -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo", "string_flag") +load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("//python/private:enum.bzl", "enum") load(":env_marker_info.bzl", "EnvMarkerInfo") load( @@ -53,64 +53,16 @@ UniversalWhlFlag = enum( UNIVERSAL = "universal", ) -_STRING_FLAGS = [ - "dist", - "whl_plat", - "whl_plat_py3", - "whl_plat_py3_abi3", - "whl_plat_pycp3x", - "whl_plat_pycp3x_abi3", - "whl_plat_pycp3x_abicp", - "whl_py3", - "whl_py3_abi3", - "whl_pycp3x", - "whl_pycp3x_abi3", - "whl_pycp3x_abicp", -] - -INTERNAL_FLAGS = [ - "whl", -] + _STRING_FLAGS - def define_pypi_internal_flags(name): """define internal PyPI flags used in PyPI hub repository by pkg_aliases. Args: name: not used """ - for flag in _STRING_FLAGS: - string_flag( - name = "_internal_pip_" + flag, - build_setting_default = "", - values = [""], - visibility = ["//visibility:public"], - ) - - _allow_wheels_flag( - name = "_internal_pip_whl", - visibility = ["//visibility:public"], - ) - _default_env_marker_config( name = "_pip_env_marker_default_config", ) -def _allow_wheels_flag_impl(ctx): - input = ctx.attr._setting[BuildSettingInfo].value - value = "yes" if input in ["auto", "only"] else "no" - return [config_common.FeatureFlagInfo(value = value)] - -_allow_wheels_flag = rule( - implementation = _allow_wheels_flag_impl, - attrs = { - "_setting": attr.label(default = "//python/config_settings:pip_whl"), - }, - doc = """\ -This rule allows us to greatly reduce the number of config setting targets at no cost even -if we are duplicating some of the functionality of the `native.config_setting`. -""", -) - def _default_env_marker_config(**kwargs): _env_marker_config( os_name = select(os_name_select_map), diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 75f3ec98d7..5b46aeeb61 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -144,8 +144,6 @@ def _whl_config_setting_dict(a): ret = {} if a.config_setting: ret["config_setting"] = a.config_setting - if a.filename: - ret["filename"] = a.filename if a.target_platforms: ret["target_platforms"] = a.target_platforms if a.version: diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index 9c610f11d3..557fc286ab 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -31,13 +31,14 @@ load("//python/private:repo_utils.bzl", "repo_utils") load(":index_sources.bzl", "index_sources") load(":parse_requirements_txt.bzl", "parse_requirements_txt") load(":pep508_requirement.bzl", "requirement") -load(":whl_target_platforms.bzl", "select_whls") +load(":select_whl.bzl", "select_whl") def parse_requirements( ctx, *, requirements_by_platform = {}, extra_pip_args = [], + platforms = {}, get_index_urls = None, evaluate_markers = None, extract_url_srcs = True, @@ -46,6 +47,7 @@ def parse_requirements( Args: ctx: A context that has .read function that would read contents from a label. + platforms: The target platform descriptions. requirements_by_platform (label_keyed_string_dict): a way to have different package versions (or different packages) for different os, arch combinations. @@ -88,7 +90,7 @@ def parse_requirements( requirements = {} for file, plats in requirements_by_platform.items(): if logger: - logger.debug(lambda: "Using {} for {}".format(file, plats)) + logger.trace(lambda: "Using {} for {}".format(file, plats)) contents = ctx.read(file) # Parse the requirements file directly in starlark to get the information @@ -161,7 +163,7 @@ def parse_requirements( # VCS package references. env_marker_target_platforms = evaluate_markers(ctx, reqs_with_env_markers) if logger: - logger.debug(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format( + logger.trace(lambda: "Evaluated env markers from:\n{}\n\nTo:\n{}".format( reqs_with_env_markers, env_marker_target_platforms, )) @@ -196,6 +198,7 @@ def parse_requirements( name = name, reqs = reqs, index_urls = index_urls, + platforms = platforms, env_marker_target_platforms = env_marker_target_platforms, extract_url_srcs = extract_url_srcs, logger = logger, @@ -203,7 +206,7 @@ def parse_requirements( ) ret.append(item) if not item.is_exposed and logger: - logger.debug(lambda: "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}".format( + logger.trace(lambda: "Package '{}' will not be exposed because it is only present on a subset of platforms: {} out of {}".format( name, sorted(requirement_target_platforms), sorted(requirements), @@ -219,38 +222,43 @@ def _package_srcs( name, reqs, index_urls, + platforms, logger, env_marker_target_platforms, extract_url_srcs): """A function to return sources for a particular package.""" srcs = {} for r in sorted(reqs.values(), key = lambda r: r.requirement_line): - whls, sdist = _add_dists( - requirement = r, - index_urls = index_urls.get(name), - logger = logger, - ) - target_platforms = env_marker_target_platforms.get(r.requirement_line, r.target_platforms) - target_platforms = sorted(target_platforms) + extra_pip_args = tuple(r.extra_pip_args) - all_dists = [] + whls - if sdist: - all_dists.append(sdist) + for target_platform in target_platforms: + if platforms and target_platform not in platforms: + fail("The target platform '{}' could not be found in {}".format( + target_platform, + platforms.keys(), + )) - if extract_url_srcs and all_dists: - req_line = r.srcs.requirement - else: - all_dists = [struct( - url = "", - filename = "", - sha256 = "", - yanked = False, - )] - req_line = r.srcs.requirement_line + dist = _add_dists( + requirement = r, + target_platform = platforms.get(target_platform), + index_urls = index_urls.get(name), + logger = logger, + ) + if logger: + logger.debug(lambda: "The whl dist is: {}".format(dist.filename if dist else dist)) + + if extract_url_srcs and dist: + req_line = r.srcs.requirement + else: + dist = struct( + url = "", + filename = "", + sha256 = "", + yanked = False, + ) + req_line = r.srcs.requirement_line - extra_pip_args = tuple(r.extra_pip_args) - for dist in all_dists: key = ( dist.filename, req_line, @@ -269,9 +277,9 @@ def _package_srcs( yanked = dist.yanked, ), ) - for p in target_platforms: - if p not in entry.target_platforms: - entry.target_platforms.append(p) + + if target_platform not in entry.target_platforms: + entry.target_platforms.append(target_platform) return srcs.values() @@ -325,7 +333,7 @@ def host_platform(ctx): repo_utils.get_platforms_cpu_name(ctx), ) -def _add_dists(*, requirement, index_urls, logger = None): +def _add_dists(*, requirement, index_urls, target_platform, logger = None): """Populate dists based on the information from the PyPI index. This function will modify the given requirements_by_platform data structure. @@ -333,6 +341,7 @@ def _add_dists(*, requirement, index_urls, logger = None): Args: requirement: The result of parse_requirements function. index_urls: The result of simpleapi_download. + target_platform: The target_platform information. logger: A logger for printing diagnostic info. """ @@ -342,7 +351,7 @@ def _add_dists(*, requirement, index_urls, logger = None): logger.debug(lambda: "Could not detect the filename from the URL, falling back to pip: {}".format( requirement.srcs.url, )) - return [], None + return None # Handle direct URLs in requirements dist = struct( @@ -353,12 +362,12 @@ def _add_dists(*, requirement, index_urls, logger = None): ) if dist.filename.endswith(".whl"): - return [dist], None + return dist else: - return [], dist + return dist if not index_urls: - return [], None + return None whls = [] sdist = None @@ -401,11 +410,16 @@ def _add_dists(*, requirement, index_urls, logger = None): for reason, dists in yanked.items() ])) - # Filter out the wheels that are incompatible with the target_platforms. - whls = select_whls( + if not target_platform: + # The pipstar platforms are undefined here, so we cannot do any matching + return sdist + + # Select a single wheel that can work on the target_platform + return select_whl( whls = whls, - want_platforms = requirement.target_platforms, + python_version = target_platform.env["python_full_version"], + implementation_name = target_platform.env["implementation_name"], + whl_abi_tags = target_platform.whl_abi_tags, + platforms = target_platform.whl_platform_tags, logger = logger, - ) - - return whls, sdist + ) or sdist diff --git a/python/private/pypi/pep508_env.bzl b/python/private/pypi/pep508_env.bzl index c2d404bc3e..a51d154219 100644 --- a/python/private/pypi/pep508_env.bzl +++ b/python/private/pypi/pep508_env.bzl @@ -15,6 +15,10 @@ """This module is for implementing PEP508 environment definition. """ +load("//python/private:version.bzl", "version") + +_DEFAULT = "//conditions:default" + # See https://stackoverflow.com/a/45125525 platform_machine_aliases = { # These pairs mean the same hardware, but different values may be used @@ -59,31 +63,25 @@ platform_machine_select_map = { "@platforms//cpu:x86_64": "x86_64", # The value is empty string if it cannot be determined: # https://docs.python.org/3/library/platform.html#platform.machine - "//conditions:default": "", + _DEFAULT: "", } # Platform system returns results from the `uname` call. -_platform_system_values = { +platform_system_select_map = { # See https://peps.python.org/pep-0738/#platform - "android": "Android", - "freebsd": "FreeBSD", + "@platforms//os:android": "Android", + "@platforms//os:freebsd": "FreeBSD", # See https://peps.python.org/pep-0730/#platform # NOTE: Per Pep 730, "iPadOS" is also an acceptable value - "ios": "iOS", - "linux": "Linux", - "netbsd": "NetBSD", - "openbsd": "OpenBSD", - "osx": "Darwin", - "windows": "Windows", -} - -platform_system_select_map = { - "@platforms//os:{}".format(bazel_os): py_system - for bazel_os, py_system in _platform_system_values.items() -} | { + "@platforms//os:ios": "iOS", + "@platforms//os:linux": "Linux", + "@platforms//os:netbsd": "NetBSD", + "@platforms//os:openbsd": "OpenBSD", + "@platforms//os:osx": "Darwin", + "@platforms//os:windows": "Windows", # The value is empty string if it cannot be determined: # https://docs.python.org/3/library/platform.html#platform.machine - "//conditions:default": "", + _DEFAULT: "", } # The copy of SO [answer](https://stackoverflow.com/a/13874620) containing @@ -111,30 +109,24 @@ platform_system_select_map = { # (**) Prior Python 3.8 could also be aix5 or aix7; use sys.platform.startswith() # # We are using only the subset that we actually support. -_sys_platform_values = { +sys_platform_select_map = { # These values are decided by the sys.platform docs. - "android": "android", - "emscripten": "emscripten", + "@platforms//os:android": "android", + "@platforms//os:emscripten": "emscripten", # NOTE: The below values are approximations. The sys.platform() docs # don't have documented values for these OSes. Per docs, the # sys.platform() value reflects the OS at the time Python was *built* # instead of the runtime (target) OS value. - "freebsd": "freebsd", - "ios": "ios", - "linux": "linux", - "openbsd": "openbsd", - "osx": "darwin", - "wasi": "wasi", - "windows": "win32", -} - -sys_platform_select_map = { - "@platforms//os:{}".format(bazel_os): py_platform - for bazel_os, py_platform in _sys_platform_values.items() -} | { + "@platforms//os:freebsd": "freebsd", + "@platforms//os:ios": "ios", + "@platforms//os:linux": "linux", + "@platforms//os:openbsd": "openbsd", + "@platforms//os:osx": "darwin", + "@platforms//os:wasi": "wasi", + "@platforms//os:windows": "win32", # For lack of a better option, use empty string. No standard doc/spec # about sys_platform value. - "//conditions:default": "", + _DEFAULT: "", } # The "java" value is documented, but with Jython defunct, @@ -142,52 +134,57 @@ sys_platform_select_map = { # The os.name value is technically a property of the runtime, not the # targetted runtime OS, but the distinction shouldn't matter if # things are properly configured. -_os_name_values = { - "linux": "posix", - "osx": "posix", - "windows": "nt", -} - os_name_select_map = { - "@platforms//os:{}".format(bazel_os): py_os - for bazel_os, py_os in _os_name_values.items() -} | { - "//conditions:default": "posix", + "@platforms//os:windows": "nt", + _DEFAULT: "posix", } -def env(target_platform, *, extra = None): +def _get_from_map(m, key): + if _DEFAULT in m: + return m.get(key, m[_DEFAULT]) + else: + return m[key] + +def env(*, env = None, os, arch, python_version = "", extra = None): """Return an env target platform NOTE: This is for use during the loading phase. For the analysis phase, `env_marker_setting()` constructs the env dict. Args: - target_platform: {type}`str` the target platform identifier, e.g. - `cp33_linux_aarch64` + env: {type}`str` the environment. + os: {type}`str` the OS name. + arch: {type}`str` the CPU name. + python_version: {type}`str` the full python version. extra: {type}`str` the extra value to be added into the env. Returns: A dict that can be used as `env` in the marker evaluation. """ - env = create_env() + env = env or {} + env = env | create_env() if extra != None: env["extra"] = extra - if target_platform.abi: - minor_version, _, micro_version = target_platform.abi[3:].partition(".") - micro_version = micro_version or "0" + if python_version: + v = version.parse(python_version) + major = v.release[0] + minor = v.release[1] + micro = v.release[2] if len(v.release) > 2 else 0 env = env | { - "implementation_version": "3.{}.{}".format(minor_version, micro_version), - "python_full_version": "3.{}.{}".format(minor_version, micro_version), - "python_version": "3.{}".format(minor_version), + "implementation_version": "{}.{}.{}".format(major, minor, micro), + "python_full_version": "{}.{}.{}".format(major, minor, micro), + "python_version": "{}.{}".format(major, minor), } - if target_platform.os and target_platform.arch: - os = target_platform.os + + if os and arch: + os = "@platforms//os:{}".format(os) + arch = "@platforms//cpu:{}".format(arch) env = env | { - "os_name": _os_name_values.get(os, ""), - "platform_machine": target_platform.arch, - "platform_system": _platform_system_values.get(os, ""), - "sys_platform": _sys_platform_values.get(os, ""), + "os_name": _get_from_map(os_name_select_map, os), + "platform_machine": _get_from_map(platform_machine_select_map, arch), + "platform_system": _get_from_map(platform_system_select_map, os), + "sys_platform": _get_from_map(sys_platform_select_map, os), } set_missing_env_defaults(env) diff --git a/python/private/pypi/pep508_platform.bzl b/python/private/pypi/pep508_platform.bzl deleted file mode 100644 index 381a8d7a08..0000000000 --- a/python/private/pypi/pep508_platform.bzl +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2025 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""The platform abstraction -""" - -def platform(*, abi = None, os = None, arch = None): - """platform returns a struct for the platform. - - Args: - abi: {type}`str | None` the target ABI, e.g. `"cp39"`. - os: {type}`str | None` the target os, e.g. `"linux"`. - arch: {type}`str | None` the target CPU, e.g. `"aarch64"`. - - Returns: - A struct. - """ - - # Note, this is used a lot as a key in dictionaries, so it cannot contain - # methods. - return struct( - abi = abi, - os = os, - arch = arch, - ) - -def platform_from_str(p, python_version): - """Return a platform from a string. - - Args: - p: {type}`str` the actual string. - python_version: {type}`str` the python version to add to platform if needed. - - Returns: - A struct that is returned by the `_platform` function. - """ - if p.startswith("cp"): - abi, _, p = p.partition("_") - elif python_version: - major, _, tail = python_version.partition(".") - abi = "cp{}{}".format(major, tail) - else: - abi = None - - os, _, arch = p.partition("_") - return platform(abi = abi, os = os or None, arch = arch or None) diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 4d3cc61590..8ebb2e254a 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -25,8 +25,6 @@ Definitions: :cpu: CPU architecture identifier that exists in `@platforms//cpu:`. :python_tag: The Python tag as defined by the [Python Packaging Authority][packaging_spec]. E.g. `py2.py3`, `py3`, `py311`, `cp311`. :abi_tag: The ABI tag as defined by the [Python Packaging Authority][packaging_spec]. E.g. `none`, `abi3`, `cp311`, `cp311t`. -:platform_tag: The Platform tag as defined by the [Python Packaging Authority][packaging_spec]. E.g. `manylinux_2_17_x86_64`. -:platform_suffix: is a derivative of the `platform_tag` and is used to implement selection based on `libc` or `osx` version. All of the config settings used by this macro are generated by {obj}`config_settings`, for more detailed documentation on what each config @@ -85,14 +83,6 @@ load( "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) -load(":parse_whl_name.bzl", "parse_whl_name") -load(":whl_target_platforms.bzl", "whl_target_platforms") - -# This value is used as sentinel value in the alias/config setting machinery -# for libc and osx versions. If we encounter this version in this part of the -# code, then it means that we have a bug in rules_python and that we should fix -# it. It is more of an internal consistency check. -_VERSION_NONE = (0, 0) _NO_MATCH_ERROR_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -219,21 +209,9 @@ def pkg_aliases( actual = "//_groups:{}_whl".format(group_name), ) -def _normalize_versions(name, versions): - if not versions: - return [] - - if _VERSION_NONE in versions: - fail("a sentinel version found in '{}', check render_pkg_aliases for bugs".format(name)) - - return sorted(versions) - def multiplatform_whl_aliases( *, - aliases = [], - glibc_versions = [], - muslc_versions = [], - osx_versions = []): + aliases = []): """convert a list of aliases from filename to config_setting ones. Exposed only for unit tests. @@ -243,12 +221,6 @@ def multiplatform_whl_aliases( to process. Any aliases that have the filename set will be converted to a dict of config settings to repo names. The struct is created by {func}`whl_config_setting`. - glibc_versions: {type}`list[tuple[int, int]]` list of versions that can be - used in this hub repo. - muslc_versions: {type}`list[tuple[int, int]]` list of versions that can be - used in this hub repo. - osx_versions: {type}`list[tuple[int, int]]` list of versions that can be - used in this hub repo. Returns: A dict with of config setting labels to repo names or the repo name itself. @@ -258,33 +230,19 @@ def multiplatform_whl_aliases( # We don't have any aliases, this is a repo name return aliases - # TODO @aignas 2024-11-17: we might be able to use FeatureFlagInfo and some - # code gen to create a version_lt_x target, which would allow us to check - # if the libc version is in a particular range. - glibc_versions = _normalize_versions("glibc_versions", glibc_versions) - muslc_versions = _normalize_versions("muslc_versions", muslc_versions) - osx_versions = _normalize_versions("osx_versions", osx_versions) - ret = {} versioned_additions = {} for alias, repo in aliases.items(): if type(alias) != "struct": ret[alias] = repo continue - elif not (alias.filename or alias.target_platforms): + elif not alias.target_platforms: # This is an internal consistency check - fail("Expected to have either 'filename' or 'target_platforms' set, got: {}".format(alias)) + fail("Expected to have 'target_platforms' set, got: {}".format(alias)) config_settings, all_versioned_settings = get_filename_config_settings( - filename = alias.filename or "", target_platforms = alias.target_platforms, python_version = alias.version, - # If we have multiple platforms but no wheel filename, lets use different - # config settings. - non_whl_prefix = "sdist" if alias.filename else "", - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, ) for setting in config_settings: @@ -331,70 +289,28 @@ def multiplatform_whl_aliases( def get_filename_config_settings( *, - filename, target_platforms, - python_version, - glibc_versions = None, - muslc_versions = None, - osx_versions = None, - non_whl_prefix = "sdist"): + python_version): """Get the filename config settings. Exposed only for unit tests. Args: - filename: the distribution filename (can be a whl or an sdist). target_platforms: list[str], target platforms in "{abi}_{os}_{cpu}" format. - glibc_versions: list[tuple[int, int]], list of versions. - muslc_versions: list[tuple[int, int]], list of versions. - osx_versions: list[tuple[int, int]], list of versions. python_version: the python version to generate the config_settings for. - non_whl_prefix: the prefix of the config setting when the whl we don't have - a filename ending with ".whl". Returns: A tuple: * A list of config settings that are generated by ./pip_config_settings.bzl * The list of default version settings. """ - prefixes = [] suffixes = [] setting_supported_versions = {} - - if filename.endswith(".whl"): - parsed = parse_whl_name(filename) - if parsed.python_tag == "py2.py3": - py = "py_" - elif parsed.python_tag == "py3": - py = "py3_" - elif parsed.python_tag.startswith("cp"): - py = "" - else: - py = "py3_" - - abi = parsed.abi_tag - - # TODO @aignas 2025-04-20: test - abi, _, _ = abi.partition(".") - - if parsed.platform_tag == "any": - prefixes = ["{}{}_any".format(py, abi)] - else: - prefixes = ["{}{}".format(py, abi)] - suffixes = _whl_config_setting_suffixes( - platform_tag = parsed.platform_tag, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, - setting_supported_versions = setting_supported_versions, - ) - else: - prefixes = [non_whl_prefix or ""] - + prefixes = [""] py = "cp{}".format(python_version).replace(".", "") prefixes = [ "{}_{}".format(py, prefix) if prefix else py - for prefix in prefixes + for prefix in [""] ] versioned = { @@ -417,49 +333,6 @@ def get_filename_config_settings( else: return [":is_{}".format(p) for p in prefixes], setting_supported_versions -def _whl_config_setting_suffixes( - platform_tag, - glibc_versions, - muslc_versions, - osx_versions, - setting_supported_versions): - suffixes = [] - for platform_tag in platform_tag.split("."): - for p in whl_target_platforms(platform_tag): - prefix = p.os - suffix = p.cpu - if "manylinux" in platform_tag: - prefix = "manylinux" - versions = glibc_versions - elif "musllinux" in platform_tag: - prefix = "musllinux" - versions = muslc_versions - elif p.os in ["linux", "windows"]: - versions = [(0, 0)] - elif p.os == "osx": - versions = osx_versions - if "universal2" in platform_tag: - suffix = "universal2" - else: - fail("Unsupported whl os: {}".format(p.os)) - - default_version_setting = "{}_{}".format(prefix, suffix) - supported_versions = {} - for v in versions: - if v == (0, 0): - suffixes.append(default_version_setting) - elif v >= p.version: - supported_versions[v] = "{}_{}_{}_{}".format( - prefix, - v[0], - v[1], - suffix, - ) - if supported_versions: - setting_supported_versions[default_version_setting] = supported_versions - - return suffixes - def _non_versioned_platform(p, *, strict = False): """A small utility function that converts 'cp311_linux_x86_64' to 'linux_x86_64'. diff --git a/python/private/pypi/python_tag.bzl b/python/private/pypi/python_tag.bzl new file mode 100644 index 0000000000..224c5f96f0 --- /dev/null +++ b/python/private/pypi/python_tag.bzl @@ -0,0 +1,41 @@ +"A simple utility function to get the python_tag from the implementation name" + +load("//python/private:version.bzl", "version") + +# Taken from +# https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#python-tag +_PY_TAGS = { + # "py": Generic Python (does not require implementation-specific features) + "cpython": "cp", + "ironpython": "ip", + "jython": "jy", + "pypy": "pp", + "python": "py", +} +PY_TAG_GENERIC = "py" + +def python_tag(implementation_name, python_version = ""): + """Get the python_tag from the implementation_name. + + Args: + implementation_name: {type}`str` the implementation name, e.g. "cpython" + python_version: {type}`str` a version who can be parsed using PEP440 compliant + parser. + + Returns: + A {type}`str` that represents the python_tag with a version if the + python_version is given. + """ + if python_version: + v = version.parse(python_version, strict = True) + suffix = "{}{}".format( + v.release[0], + v.release[1] if len(v.release) > 1 else "", + ) + else: + suffix = "" + + return "{}{}".format( + _PY_TAGS.get(implementation_name, implementation_name), + suffix, + ) diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index e743fc20f7..77063921c8 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -22,8 +22,6 @@ load( ":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel", ) # buildifier: disable=bzl-visibility -load(":parse_whl_name.bzl", "parse_whl_name") -load(":whl_target_platforms.bzl", "whl_target_platforms") NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -48,11 +46,10 @@ def _repr_dict(*, value_repr = repr, **kwargs): return {k: value_repr(v) for k, v in kwargs.items() if v} def _repr_config_setting(alias): - if alias.filename or alias.target_platforms: + if alias.target_platforms: return render.call( "whl_config_setting", **_repr_dict( - filename = alias.filename, target_platforms = alias.target_platforms, config_setting = alias.config_setting, version = alias.version, @@ -179,15 +176,9 @@ def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, contents = render_pkg_aliases( aliases = aliases, - glibc_versions = flag_versions.get("glibc_versions", []), - muslc_versions = flag_versions.get("muslc_versions", []), - osx_versions = flag_versions.get("osx_versions", []), **kwargs ) contents["_config/BUILD.bazel"] = _render_config_settings( - glibc_versions = flag_versions.get("glibc_versions", []), - muslc_versions = flag_versions.get("muslc_versions", []), - osx_versions = flag_versions.get("osx_versions", []), python_versions = _major_minor_versions(flag_versions.get("python_versions", [])), platform_config_settings = platform_config_settings, visibility = ["//:__subpackages__"], @@ -219,54 +210,21 @@ def get_whl_flag_versions(settings): * python_versions """ python_versions = {} - glibc_versions = {} target_platforms = {} - muslc_versions = {} - osx_versions = {} for setting in settings: - if not setting.version and not setting.filename: + if not setting.version: continue if setting.version: python_versions[setting.version] = None - if setting.filename and setting.filename.endswith(".whl") and not setting.filename.endswith("-any.whl"): - parsed = parse_whl_name(setting.filename) - else: - for plat in setting.target_platforms or []: - target_platforms[_non_versioned_platform(plat)] = None - continue - - for platform_tag in parsed.platform_tag.split("."): - parsed = whl_target_platforms(platform_tag) - - for p in parsed: - target_platforms[p.target_platform] = None - - if platform_tag.startswith("win") or platform_tag.startswith("linux"): - continue - - head, _, tail = platform_tag.partition("_") - major, _, tail = tail.partition("_") - minor, _, tail = tail.partition("_") - if tail: - version = (int(major), int(minor)) - if "many" in head: - glibc_versions[version] = None - elif "musl" in head: - muslc_versions[version] = None - elif "mac" in head: - osx_versions[version] = None - else: - fail(platform_tag) + for plat in setting.target_platforms or []: + target_platforms[_non_versioned_platform(plat)] = None return { k: sorted(v) for k, v in { - "glibc_versions": glibc_versions, - "muslc_versions": muslc_versions, - "osx_versions": osx_versions, "python_versions": python_versions, "target_platforms": target_platforms, }.items() diff --git a/python/private/pypi/select_whl.bzl b/python/private/pypi/select_whl.bzl new file mode 100644 index 0000000000..395b8ade96 --- /dev/null +++ b/python/private/pypi/select_whl.bzl @@ -0,0 +1,164 @@ +"Select a single wheel that fits the parameters of a target platform." + +load("//python/private:version.bzl", "version") +load(":parse_whl_name.bzl", "parse_whl_name") +load(":python_tag.bzl", "PY_TAG_GENERIC", "python_tag") + +def _priority_by_values(*, tag, values, allow_wildcard = True): + keys = [] + for priority, wp in enumerate(values): + head, sep, tail = wp.partition("*") + if "*" in tail: + fail("only a single '*' can be present in the matcher") + if not allow_wildcard and sep: + fail("'*' is not allowed in the matcher") + + if not sep and tag == head: + keys.append(priority) + elif sep and tag.startswith(head) and tag.endswith(tail): + keys.append(priority) + + return max(keys) if keys else None + +def _priority_by_version(*, tag, implementation, py_version): + if tag.startswith(PY_TAG_GENERIC): + ver_str = tag[len(PY_TAG_GENERIC):] + elif tag.startswith(implementation): + ver_str = tag[len(implementation):] + else: + return None + + # Add a 0 at the end in case it is a single digit + ver_str = "{}.{}".format(ver_str[0], ver_str[1:] or "0") + + ver = version.parse(ver_str) + if not version.is_compatible(py_version, ver): + return None + + return ( + tag.startswith(implementation), + version.key(ver), + ) + +def _candidates_by_priority(*, whls, implementation, py_version, whl_abi_tags, platforms, logger): + """Calculate the priority of each wheel + + Args: + whls: {type}`list[struct]` The whls to select from. + implementation: {type}`str` The target Python implementation. + py_version: {type}`struct` The target python version. + whl_abi_tags: {type}`list[str]` The whl abi tags to select from. + platforms: {type}`list[str]` The whl platform tags to select from. + logger: The logger to use for debugging info + + Returns: + A dictionary where keys are priority tuples which allows us to sort and pick the + last item. + """ + + ret = {} + for whl in whls: + parsed = parse_whl_name(whl.filename) + priority = None + + # See https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/#compressed-tag-sets + for platform in parsed.platform_tag.split("."): + platform = _priority_by_values(tag = platform, values = platforms) + if platform == None: + if logger: + logger.debug(lambda: "The platform_tag in '{}' does not match given list: {}".format( + whl.filename, + platforms, + )) + continue + + for py in parsed.python_tag.split("."): + py = _priority_by_version( + tag = py, + implementation = implementation, + py_version = py_version, + ) + if py == None: + if logger: + logger.debug(lambda: "The python_tag in '{}' does not match implementation or version: {} {}".format( + whl.filename, + implementation, + py_version.string, + )) + continue + + for abi in parsed.abi_tag.split("."): + abi = _priority_by_values( + tag = abi, + values = whl_abi_tags, + allow_wildcard = False, + ) + if abi == None: + if logger: + logger.debug(lambda: "The abi_tag in '{}' does not match given list: {}".format( + whl.filename, + whl_abi_tags, + )) + continue + + # 1. Prefer platform wheels + # 2. Then prefer implementation/python version + # 3. Then prefer more specific ABI wheels + candidate = (platform, py, abi) + priority = priority or candidate + if candidate > priority: + priority = candidate + + if priority == None: + if logger: + logger.debug(lambda: "The whl '{}' is incompatible".format( + whl.filename, + )) + continue + + ret[priority] = whl + + return ret + +def select_whl(*, whls, python_version, platforms, whl_abi_tags, implementation_name = "cpython", limit = 1, logger = None): + """Select a whl that is the most suitable for the given platform. + + Args: + whls: {type}`list[struct]` a list of candidates which have a `filename` + attribute containing the `whl` filename. + python_version: {type}`str` the target python version. + platforms: {type}`list[str]` the target platform identifiers that may contain + a single `*` character. + implementation_name: {type}`str` the `implementation_name` from the target_platform env. + whl_abi_tags: {type}`str` the ABIs that the target_platform is compatible with. + limit: {type}`int` number of wheels to return. Defaults to 1. + logger: {type}`struct` the logger instance. + + Returns: + {type}`list[struct] | struct | None`, a single struct from the `whls` input + argument or `None` if a match is not found. If the `limit` is greater than + one, then we will return a list. + """ + py_version = version.parse(python_version, strict = True) + candidates = {} + implementation = python_tag(implementation_name) + + candidates = _candidates_by_priority( + whls = whls, + implementation = implementation, + py_version = py_version, + whl_abi_tags = whl_abi_tags, + platforms = platforms, + logger = logger, + ) + + if not candidates: + return None + + res = [i[1] for i in sorted(candidates.items())] + if logger: + logger.debug(lambda: "Sorted candidates:\n{}".format( + "\n".join([c.filename for c in res]), + )) + + return res[-1] if limit == 1 else res[-limit:] diff --git a/python/private/pypi/whl_config_setting.bzl b/python/private/pypi/whl_config_setting.bzl index 3b81e4694f..da6dd80dc1 100644 --- a/python/private/pypi/whl_config_setting.bzl +++ b/python/private/pypi/whl_config_setting.bzl @@ -14,7 +14,7 @@ "A small function to create an alias for a whl distribution" -def whl_config_setting(*, version = None, config_setting = None, filename = None, target_platforms = None): +def whl_config_setting(*, version = None, config_setting = None, target_platforms = None): """The bzl_packages value used by by the render_pkg_aliases function. This contains the minimum amount of information required to generate correct @@ -27,7 +27,6 @@ def whl_config_setting(*, version = None, config_setting = None, filename = None is no match found during a select. config_setting: {type}`str | Label | None` the config setting that we should use. Defaults to "//_config:is_python_{version}". - filename: {type}`str | None` the distribution filename to derive the config_setting. target_platforms: {type}`list[str] | None` the list of target_platforms for this distribution. @@ -51,7 +50,6 @@ def whl_config_setting(*, version = None, config_setting = None, filename = None return struct( config_setting = config_setting, - filename = filename, # Make the struct hashable target_platforms = tuple(target_platforms) if target_platforms else None, version = version, diff --git a/python/private/pypi/whl_target_platforms.bzl b/python/private/pypi/whl_target_platforms.bzl index 6ea3f120c3..6c3dd5da83 100644 --- a/python/private/pypi/whl_target_platforms.bzl +++ b/python/private/pypi/whl_target_platforms.bzl @@ -16,8 +16,6 @@ A starlark implementation of the wheel platform tag parsing to get the target platform. """ -load(":parse_whl_name.bzl", "parse_whl_name") - # The order of the dictionaries is to keep definitions with their aliases next to each # other _CPU_ALIASES = { @@ -46,136 +44,6 @@ _OS_PREFIXES = { "win": "windows", } # buildifier: disable=unsorted-dict-items -def select_whls(*, whls, want_platforms = [], logger = None): - """Select a subset of wheels suitable for target platforms from a list. - - Args: - whls(list[struct]): A list of candidates which have a `filename` - attribute containing the `whl` filename. - want_platforms(str): The platforms in "{abi}_{os}_{cpu}" or "{os}_{cpu}" format. - logger: A logger for printing diagnostic messages. - - Returns: - A filtered list of items from the `whls` arg where `filename` matches - the selected criteria. If no match is found, an empty list is returned. - """ - if not whls: - return [] - - want_abis = { - "abi3": None, - "none": None, - } - - _want_platforms = {} - version_limit = None - - for p in want_platforms: - if not p.startswith("cp3"): - fail("expected all platforms to start with ABI, but got: {}".format(p)) - - abi, _, os_cpu = p.partition("_") - abi, _, _ = abi.partition(".") - _want_platforms[os_cpu] = None - - # TODO @aignas 2025-04-20: add a test - _want_platforms["{}_{}".format(abi, os_cpu)] = None - - version_limit_candidate = int(abi[3:]) - if not version_limit: - version_limit = version_limit_candidate - if version_limit and version_limit != version_limit_candidate: - fail("Only a single python version is supported for now") - - # For some legacy implementations the wheels may target the `cp3xm` ABI - _want_platforms["{}m_{}".format(abi, os_cpu)] = None - want_abis[abi] = None - want_abis[abi + "m"] = None - - # Also add freethreaded wheels if we find them since we started supporting them - _want_platforms["{}t_{}".format(abi, os_cpu)] = None - want_abis[abi + "t"] = None - - want_platforms = sorted(_want_platforms) - - candidates = {} - for whl in whls: - parsed = parse_whl_name(whl.filename) - - if logger: - logger.trace(lambda: "Deciding whether to use '{}'".format(whl.filename)) - - supported_implementations = {} - whl_version_min = 0 - for tag in parsed.python_tag.split("."): - supported_implementations[tag[:2]] = None - - if tag.startswith("cp3") or tag.startswith("py3"): - version = int(tag[len("..3"):] or 0) - else: - # In this case it should be eithor "cp2" or "py2" and we will default - # to `whl_version_min` = 0 - continue - - if whl_version_min == 0 or version < whl_version_min: - whl_version_min = version - - if not ("cp" in supported_implementations or "py" in supported_implementations): - if logger: - logger.trace(lambda: "Discarding the whl because the whl does not support CPython, whl supported implementations are: {}".format(supported_implementations)) - continue - - if want_abis and parsed.abi_tag not in want_abis: - # Filter out incompatible ABIs - if logger: - logger.trace(lambda: "Discarding the whl because the whl abi did not match") - continue - - if whl_version_min > version_limit: - if logger: - logger.trace(lambda: "Discarding the whl because the whl supported python version is too high") - continue - - compatible = False - if parsed.platform_tag == "any": - compatible = True - else: - for p in whl_target_platforms(parsed.platform_tag, abi_tag = parsed.abi_tag.strip("m") if parsed.abi_tag.startswith("cp") else None): - if p.target_platform in want_platforms: - compatible = True - break - - if not compatible: - if logger: - logger.trace(lambda: "Discarding the whl because the whl does not support the desired platforms: {}".format(want_platforms)) - continue - - for implementation in supported_implementations: - candidates.setdefault( - ( - parsed.abi_tag, - parsed.platform_tag, - ), - {}, - ).setdefault( - ( - # prefer cp implementation - implementation == "cp", - # prefer higher versions - whl_version_min, - # prefer abi3 over none - parsed.abi_tag != "none", - # prefer cpx abi over abi3 - parsed.abi_tag != "abi3", - ), - [], - ).append(whl) - - return [ - candidates[key][sorted(v)[-1]][-1] - for key, v in candidates.items() - ] - def whl_target_platforms(platform_tag, abi_tag = ""): """Parse the wheel abi and platform tags and return (os, cpu) tuples. diff --git a/tests/pypi/config_settings/config_settings_tests.bzl b/tests/pypi/config_settings/config_settings_tests.bzl index a15f6b4d32..3f2dbd26e7 100644 --- a/tests/pypi/config_settings/config_settings_tests.bzl +++ b/tests/pypi/config_settings/config_settings_tests.bzl @@ -78,9 +78,9 @@ def _test_legacy_default(name): _analysis_test( name = name, dist = { - "is_cp37": "legacy", + "is_cp37": "foo", }, - want = "legacy", + want = "foo", ) _tests.append(_test_legacy_default) @@ -89,562 +89,13 @@ def _test_legacy_with_constraint_values(name): _analysis_test( name = name, dist = { - "is_cp37": "legacy", - "is_cp37_linux_aarch64": "legacy_platform_override", + "is_cp37_linux_aarch64": "foo", }, - want = "legacy_platform_override", + want = "foo", ) _tests.append(_test_legacy_with_constraint_values) -# Tests when we only have an `sdist` present. - -def _test_sdist_default(name): - _analysis_test( - name = name, - dist = { - "is_cp37_sdist": "sdist", - }, - want = "sdist", - ) - -_tests.append(_test_sdist_default) - -def _test_legacy_less_specialized_than_sdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37": "legacy", - "is_cp37_sdist": "sdist", - }, - want = "sdist", - ) - -_tests.append(_test_legacy_less_specialized_than_sdist) - -def _test_sdist_no_whl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("no"), - ], - want = "sdist", - ) - -_tests.append(_test_sdist_no_whl) - -def _test_sdist_no_sdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("only"), - ], - # We will use `no_match_error` in the real case to indicate that `sdist` is not - # allowed to be used. - want = "no_match", - ) - -_tests.append(_test_sdist_no_sdist) - -def _test_basic_whl_default(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py_none_any": "whl", - "is_cp37_sdist": "sdist", - }, - want = "whl", - ) - -_tests.append(_test_basic_whl_default) - -def _test_basic_whl_nowhl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py_none_any": "whl", - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("no"), - ], - want = "sdist", - ) - -_tests.append(_test_basic_whl_nowhl) - -def _test_basic_whl_nosdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py_none_any": "whl", - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("only"), - ], - want = "whl", - ) - -_tests.append(_test_basic_whl_nosdist) - -def _test_whl_default(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "whl", - "is_cp37_py_none_any": "basic_whl", - }, - want = "whl", - ) - -_tests.append(_test_whl_default) - -def _test_whl_nowhl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "whl", - "is_cp37_py_none_any": "basic_whl", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("no"), - ], - want = "no_match", - ) - -_tests.append(_test_whl_nowhl) - -def _test_whl_nosdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "whl", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("only"), - ], - want = "whl", - ) - -_tests.append(_test_whl_nosdist) - -def _test_abi_whl_is_prefered(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_abi3_any": "abi_whl", - "is_cp37_py3_none_any": "whl", - }, - want = "abi_whl", - ) - -_tests.append(_test_abi_whl_is_prefered) - -def _test_whl_with_constraints_is_prefered(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "default_whl", - "is_cp37_py3_none_any_linux_aarch64": "whl", - "is_cp37_py3_none_any_linux_x86_64": "amd64_whl", - }, - want = "whl", - ) - -_tests.append(_test_whl_with_constraints_is_prefered) - -def _test_cp_whl_is_prefered_over_py3(name): - _analysis_test( - name = name, - dist = { - "is_cp37_none_any": "cp", - "is_cp37_py3_abi3_any": "py3_abi3", - "is_cp37_py3_none_any": "py3", - }, - want = "cp", - ) - -_tests.append(_test_cp_whl_is_prefered_over_py3) - -def _test_cp_abi_whl_is_prefered_over_py3(name): - _analysis_test( - name = name, - dist = { - "is_cp37_abi3_any": "cp", - "is_cp37_py3_abi3_any": "py3", - }, - want = "cp", - ) - -_tests.append(_test_cp_abi_whl_is_prefered_over_py3) - -def _test_cp_version_is_selected_when_python_version_is_specified(name): - _analysis_test( - name = name, - dist = { - "is_cp310_none_any": "cp310", - "is_cp38_none_any": "cp38", - "is_cp39_none_any": "cp39", - }, - want = "cp310", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_cp_version_is_selected_when_python_version_is_specified) - -def _test_py_none_any_versioned(name): - _analysis_test( - name = name, - dist = { - "is_cp310_py_none_any": "whl", - "is_cp39_py_none_any": "too-low", - }, - want = "whl", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_py_none_any_versioned) - -def _test_cp_whl_is_not_prefered_over_py3_non_freethreaded(name): - _analysis_test( - name = name, - dist = { - "is_cp37_abi3_any": "py3_abi3", - "is_cp37_cp37t_any": "cp", - "is_cp37_none_any": "py3", - }, - want = "py3_abi3", - config_settings = [ - _flag.py_freethreaded("no"), - ], - ) - -_tests.append(_test_cp_whl_is_not_prefered_over_py3_non_freethreaded) - -def _test_cp_whl_is_not_prefered_over_py3_freethreaded(name): - _analysis_test( - name = name, - dist = { - "is_cp37_abi3_any": "py3_abi3", - "is_cp37_cp37_any": "cp", - "is_cp37_none_any": "py3", - }, - want = "py3", - config_settings = [ - _flag.py_freethreaded("yes"), - ], - ) - -_tests.append(_test_cp_whl_is_not_prefered_over_py3_freethreaded) - -def _test_cp_cp_whl(name): - _analysis_test( - name = name, - dist = { - "is_cp310_cp310_linux_aarch64": "whl", - }, - want = "whl", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_cp_cp_whl) - -def _test_cp_version_sdist_is_selected(name): - _analysis_test( - name = name, - dist = { - "is_cp310_sdist": "sdist", - }, - want = "sdist", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_cp_version_sdist_is_selected) - -# NOTE: Right now there is no way to get the following behaviour without -# breaking other tests. We need to choose either ta have the correct -# specialization behaviour between `is_cp37_cp37_any` and -# `is_cp37_cp37_any_linux_aarch64` or this commented out test case. -# -# I think having this behaviour not working is fine because the `suffix` -# will be either present on all of config settings of the same platform -# or none, because we use it as a way to select a separate version of the -# wheel for a single platform only. -# -# If we can think of a better way to handle it, then we can lift this -# limitation. -# -# def _test_any_whl_with_suffix_specialization(name): -# _analysis_test( -# name = name, -# dist = { -# "is_cp37_abi3_any_linux_aarch64": "abi3", -# "is_cp37_cp37_any": "cp37", -# }, -# want = "cp37", -# ) -# -# _tests.append(_test_any_whl_with_suffix_specialization) - -def _test_platform_vs_any_with_suffix_specialization(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_any_linux_aarch64": "any", - "is_cp37_py3_none_linux_aarch64": "platform_whl", - }, - want = "platform_whl", - ) - -_tests.append(_test_platform_vs_any_with_suffix_specialization) - -def _test_platform_whl_is_prefered_over_any_whl_with_constraints(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_abi3_any": "better_default_whl", - "is_cp37_py3_abi3_any_linux_aarch64": "better_default_any_whl", - "is_cp37_py3_none_any": "default_whl", - "is_cp37_py3_none_any_linux_aarch64": "whl", - "is_cp37_py3_none_linux_aarch64": "platform_whl", - }, - want = "platform_whl", - ) - -_tests.append(_test_platform_whl_is_prefered_over_any_whl_with_constraints) - -def _test_abi3_platform_whl_preference(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_abi3_linux_aarch64": "abi3_platform", - "is_cp37_py3_none_linux_aarch64": "platform", - }, - want = "abi3_platform", - ) - -_tests.append(_test_abi3_platform_whl_preference) - -def _test_glibc(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_manylinux_aarch64": "glibc", - "is_cp37_py3_abi3_linux_aarch64": "abi3_platform", - }, - want = "glibc", - ) - -_tests.append(_test_glibc) - -def _test_glibc_versioned(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_manylinux_2_14_aarch64": "glibc", - "is_cp37_cp37_manylinux_2_17_aarch64": "glibc", - "is_cp37_py3_abi3_linux_aarch64": "abi3_platform", - }, - want = "glibc", - config_settings = [ - _flag.py_linux_libc("glibc"), - _flag.pip_whl_glibc_version("2.17"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_glibc_versioned) - -def _test_glibc_compatible_exists(name): - _analysis_test( - name = name, - dist = { - # Code using the conditions will need to construct selects, which - # do the version matching correctly. - "is_cp37_cp37_manylinux_2_14_aarch64": "2_14_whl_via_2_14_branch", - "is_cp37_cp37_manylinux_2_17_aarch64": "2_14_whl_via_2_17_branch", - }, - want = "2_14_whl_via_2_17_branch", - config_settings = [ - _flag.py_linux_libc("glibc"), - _flag.pip_whl_glibc_version("2.17"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_glibc_compatible_exists) - -def _test_musl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_musllinux_aarch64": "musl", - }, - want = "musl", - config_settings = [ - _flag.py_linux_libc("musl"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_musl) - -def _test_windows(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_windows_x86_64": "whl", - "is_cp37_cp37t_windows_x86_64": "whl_freethreaded", - }, - want = "whl", - config_settings = [ - _flag.platform("windows_x86_64"), - ], - ) - -_tests.append(_test_windows) - -def _test_windows_freethreaded(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_windows_x86_64": "whl", - "is_cp37_cp37t_windows_x86_64": "whl_freethreaded", - }, - want = "whl_freethreaded", - config_settings = [ - _flag.platform("windows_x86_64"), - _flag.py_freethreaded("yes"), - ], - ) - -_tests.append(_test_windows_freethreaded) - -def _test_osx(name): - _analysis_test( - name = name, - dist = { - # We prefer arch specific whls over universal - "is_cp37_cp37_osx_universal2": "universal_whl", - "is_cp37_cp37_osx_x86_64": "whl", - }, - want = "whl", - config_settings = [ - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx) - -def _test_osx_universal_default(name): - _analysis_test( - name = name, - dist = { - # We default to universal if only that exists - "is_cp37_cp37_osx_universal2": "whl", - }, - want = "whl", - config_settings = [ - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx_universal_default) - -def _test_osx_universal_only(name): - _analysis_test( - name = name, - dist = { - # If we prefer universal, then we use that - "is_cp37_cp37_osx_universal2": "universal", - "is_cp37_cp37_osx_x86_64": "whl", - }, - want = "universal", - config_settings = [ - _flag.pip_whl_osx_arch("universal"), - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx_universal_only) - -def _test_osx_os_version(name): - _analysis_test( - name = name, - dist = { - # Similarly to the libc version, the user of the config settings will have to - # construct the select so that the version selection is correct. - "is_cp37_cp37_osx_10_9_x86_64": "whl", - }, - want = "whl", - config_settings = [ - _flag.pip_whl_osx_version("10.9"), - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx_os_version) - -def _test_all(name): - _analysis_test( - name = name, - dist = { - "is_cp37_" + f: f - for f in [ - "{py}{abi}_{plat}".format(py = valid_py, abi = valid_abi, plat = valid_plat) - # we have py2.py3, py3, cp3 - for valid_py in ["py_", "py3_", ""] - # cp abi usually comes with a version and we only need one - # config setting variant for all of them because the python - # version will discriminate between different versions. - for valid_abi in ["none", "abi3", "cp37"] - for valid_plat in [ - "any", - "manylinux_2_17_x86_64", - "manylinux_2_17_aarch64", - "osx_x86_64", - "windows_x86_64", - ] - if not ( - valid_abi == "abi3" and valid_py == "py_" or - valid_abi == "cp37" and valid_py != "" - ) - ] - }, - want = "cp37_manylinux_2_17_x86_64", - config_settings = [ - _flag.pip_whl_glibc_version("2.17"), - _flag.platform("linux_x86_64"), - ], - ) - -_tests.append(_test_all) - def config_settings_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, @@ -654,9 +105,6 @@ def config_settings_test_suite(name): # buildifier: disable=function-docstring config_settings( name = "dummy", python_versions = ["3.7", "3.8", "3.9", "3.10"], - glibc_versions = [(2, 14), (2, 17)], - muslc_versions = [(1, 1)], - osx_versions = [(10, 9), (11, 0)], platform_config_settings = { "linux_aarch64": [ "@platforms//cpu:aarch64", diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 0303843e80..2c22f2ad48 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -65,13 +65,14 @@ def _mod(*, name, default = [], parse = [], override = [], whl_mods = [], is_roo "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), ], + whl_platform_tags = whl_platform_tags, ) - for os, cpu in [ - ("linux", "x86_64"), - ("linux", "aarch64"), - ("osx", "aarch64"), - ("windows", "aarch64"), - ] + for (os, cpu), whl_platform_tags in { + ("linux", "x86_64"): ["linux_*_x86_64", "manylinux_*_x86_64"], + ("linux", "aarch64"): ["linux_*_aarch64", "manylinux_*_aarch64"], + ("osx", "aarch64"): ["macosx_*_arm64"], + ("windows", "aarch64"): ["win_arm64"], + }.items() ], ), is_root = is_root, @@ -93,21 +94,22 @@ def _parse_modules(env, enable_pipstar = 0, **kwargs): ) def _default( + *, arch_name = None, config_settings = None, os_name = None, platform = None, + whl_platform_tags = None, env = None, - whl_limit = None, - whl_platforms = None): + whl_abi_tags = None): return struct( arch_name = arch_name, os_name = os_name, platform = platform, + whl_platform_tags = whl_platform_tags or [], config_settings = config_settings, env = env or {}, - whl_platforms = whl_platforms, - whl_limit = whl_limit, + whl_abi_tags = whl_abi_tags or [], ) def _parse( @@ -405,18 +407,21 @@ def _test_torch_experimental_index_url(env): "@platforms//os:{}".format(os), "@platforms//cpu:{}".format(cpu), ], + whl_platform_tags = whl_platform_tags, ) - for os, cpu in [ - ("linux", "aarch64"), - ("linux", "x86_64"), - ("osx", "aarch64"), - ("windows", "x86_64"), - ] + for (os, cpu), whl_platform_tags in { + ("linux", "x86_64"): ["linux_*_x86_64", "manylinux_*_x86_64"], + ("linux", "aarch64"): ["linux_*_aarch64", "manylinux_*_aarch64"], + ("osx", "aarch64"): ["macosx_*_arm64"], + ("windows", "x86_64"): ["win_amd64"], + ("windows", "aarch64"): ["win_arm64"], # this should be ignored + }.items() ], parse = [ _parse( hub_name = "pypi", python_version = "3.12", + download_only = True, experimental_index_url = "https://torch.index", requirements_lock = "universal.txt", ), @@ -473,25 +478,25 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "torch": { "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [ whl_config_setting( - filename = "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", + target_platforms = ["cp312_linux_x86_64"], version = "3.12", ), ], "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [ whl_config_setting( - filename = "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + target_platforms = ["cp312_linux_aarch64"], version = "3.12", ), ], "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [ whl_config_setting( - filename = "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", + target_platforms = ["cp312_windows_x86_64"], version = "3.12", ), ], "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [ whl_config_setting( - filename = "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", + target_platforms = ["cp312_osx_aarch64"], version = "3.12", ), ], @@ -500,10 +505,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ pypi.whl_libraries().contains_exactly({ "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_x86_64", - "windows_x86_64", - ], + "experimental_target_platforms": ["linux_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -512,10 +514,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "osx_aarch64", - ], + "experimental_target_platforms": ["linux_aarch64"], "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -524,10 +523,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_x86_64", - "windows_x86_64", - ], + "experimental_target_platforms": ["windows_x86_64"], "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1+cpu", @@ -536,10 +532,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": { "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "osx_aarch64", - ], + "experimental_target_platforms": ["osx_aarch64"], "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "torch==2.4.1", @@ -746,78 +739,79 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "pypi": { "direct_sdist_without_sha": { "pypi_315_any_name": [ - struct( - config_setting = None, - filename = "any-name.tar.gz", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "direct_without_sha": { "pypi_315_direct_without_sha_0_0_1_py3_none_any": [ - struct( - config_setting = None, - filename = "direct_without_sha-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "git_dep": { "pypi_315_git_dep": [ - struct( - config_setting = None, - filename = None, - target_platforms = None, + whl_config_setting( version = "3.15", ), ], }, "pip_fallback": { "pypi_315_pip_fallback": [ - struct( - config_setting = None, - filename = None, - target_platforms = None, + whl_config_setting( version = "3.15", ), ], }, "simple": { "pypi_315_simple_py3_none_any_deadb00f": [ - struct( - config_setting = None, - filename = "simple-0.0.1-py3-none-any.whl", - target_platforms = None, - version = "3.15", - ), - ], - "pypi_315_simple_sdist_deadbeef": [ - struct( - config_setting = None, - filename = "simple-0.0.1.tar.gz", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "some_other_pkg": { "pypi_315_some_py3_none_any_deadb33f": [ - struct( - config_setting = None, - filename = "some-other-pkg-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], }, "some_pkg": { "pypi_315_some_pkg_py3_none_any_deadbaaf": [ - struct( - config_setting = None, - filename = "some_pkg-0.0.1-py3-none-any.whl", - target_platforms = None, + whl_config_setting( + target_platforms = ( + "cp315_linux_aarch64", + "cp315_linux_x86_64", + "cp315_osx_aarch64", + "cp315_windows_aarch64", + ), version = "3.15", ), ], @@ -880,21 +874,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "sha256": "deadb00f", "urls": ["example2.org"], }, - "pypi_315_simple_sdist_deadbeef": { - "dep_template": "@pypi//{name}:{target}", - "experimental_target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "windows_aarch64", - ], - "extra_pip_args": ["--extra-args-for-sdist-building"], - "filename": "simple-0.0.1.tar.gz", - "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.1", - "sha256": "deadbeef", - "urls": ["example.org"], - }, "pypi_315_some_pkg_py3_none_any_deadbaaf": { "dep_template": "@pypi//{name}:{target}", "experimental_target_platforms": [ @@ -991,7 +970,6 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' "cp315_linux_x86_64", ], config_setting = None, - filename = None, ), ], "pypi_315_optimum_osx_aarch64": [ @@ -1001,7 +979,6 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' "cp315_osx_aarch64", ], config_setting = None, - filename = None, ), ], }, diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 82fdd0a051..f88c274ecb 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -16,6 +16,7 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private/pypi:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility +load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility def _mock_ctx(): testdata = { @@ -522,6 +523,26 @@ def _test_overlapping_shas_with_index_results(env): "requirements_linux": ["cp39_linux_x86_64"], "requirements_osx": ["cp39_osx_x86_64"], }, + platforms = { + "cp39_linux_x86_64": struct( + env = pep508_env( + python_version = "3.9.0", + os = "linux", + arch = "x86_64", + ), + whl_abi_tags = ["none"], + whl_platform_tags = ["any"], + ), + "cp39_osx_x86_64": struct( + env = pep508_env( + python_version = "3.9.0", + os = "osx", + arch = "x86_64", + ), + whl_abi_tags = ["none"], + whl_platform_tags = ["macosx_*"], + ), + }, get_index_urls = lambda _, __: { "foo": struct( sdists = { @@ -563,20 +584,10 @@ def _test_overlapping_shas_with_index_results(env): filename = "foo-0.0.1-py3-none-any.whl", requirement_line = "foo==0.0.3", sha256 = "deadbaaf", - target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"], + target_platforms = ["cp39_linux_x86_64"], url = "super2", yanked = False, ), - struct( - distribution = "foo", - extra_pip_args = [], - filename = "foo-0.0.1.tar.gz", - requirement_line = "foo==0.0.3", - sha256 = "5d15t", - target_platforms = ["cp39_linux_x86_64", "cp39_osx_x86_64"], - url = "sdist", - yanked = False, - ), struct( distribution = "foo", extra_pip_args = [], diff --git a/tests/pypi/pep508/evaluate_tests.bzl b/tests/pypi/pep508/evaluate_tests.bzl index cc867f346c..7843f88e89 100644 --- a/tests/pypi/pep508/evaluate_tests.bzl +++ b/tests/pypi/pep508/evaluate_tests.bzl @@ -16,7 +16,6 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private/pypi:pep508_env.bzl", pep508_env = "env") # buildifier: disable=bzl-visibility load("//python/private/pypi:pep508_evaluate.bzl", "evaluate", "tokenize") # buildifier: disable=bzl-visibility -load("//python/private/pypi:pep508_platform.bzl", "platform_from_str") # buildifier: disable=bzl-visibility _tests = [] @@ -244,26 +243,37 @@ _tests.append(_evaluate_partial_only_extra) def _evaluate_with_aliases(env): # When - for target_platform, tests in { + for (os, cpu), tests in { # buildifier: @unsorted-dict-items - "osx_aarch64": { + ("osx", "aarch64"): { "platform_system == 'Darwin' and platform_machine == 'arm64'": True, "platform_system == 'Darwin' and platform_machine == 'aarch64'": True, "platform_system == 'Darwin' and platform_machine == 'amd64'": False, }, - "osx_x86_64": { + ("osx", "x86_64"): { "platform_system == 'Darwin' and platform_machine == 'amd64'": True, "platform_system == 'Darwin' and platform_machine == 'x86_64'": True, }, - "osx_x86_32": { + ("osx", "x86_32"): { "platform_system == 'Darwin' and platform_machine == 'i386'": True, "platform_system == 'Darwin' and platform_machine == 'i686'": True, "platform_system == 'Darwin' and platform_machine == 'x86_32'": True, "platform_system == 'Darwin' and platform_machine == 'x86_64'": False, }, + ("freebsd", "x86_32"): { + "platform_system == 'FreeBSD' and platform_machine == 'i386'": True, + "platform_system == 'FreeBSD' and platform_machine == 'i686'": True, + "platform_system == 'FreeBSD' and platform_machine == 'x86_32'": True, + "platform_system == 'FreeBSD' and platform_machine == 'x86_64'": False, + "platform_system == 'FreeBSD' and os_name == 'posix'": True, + }, }.items(): # buildifier: @unsorted-dict-items for input, want in tests.items(): - _check_evaluate(env, input, want, pep508_env(platform_from_str(target_platform, ""))) + _check_evaluate(env, input, want, pep508_env( + os = os, + arch = cpu, + python_version = "3.2", + )) _tests.append(_evaluate_with_aliases) diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 3fd08c393c..e36f590494 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -158,15 +158,6 @@ def _test_multiplatform_whl_aliases(env): pkg_aliases( name = "bar_baz", actual = { - whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", - version = "3.9", - ): "filename_repo", - whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", - version = "3.9", - target_platforms = ["cp39_linux_x86_64"], - ): "filename_repo_for_platform", whl_config_setting( version = "3.9", target_platforms = ["cp39_linux_x86_64"], @@ -178,9 +169,6 @@ def _test_multiplatform_whl_aliases(env): alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), ), select = mock_select, - glibc_versions = [], - muslc_versions = [], - osx_versions = [], ) # buildifier: disable=unsorted-dict-items @@ -188,8 +176,6 @@ def _test_multiplatform_whl_aliases(env): "pkg": { "//:my_config_setting": "@bzlmod_repo//:pkg", "//_config:is_cp39_linux_x86_64": "@bzlmod_repo_for_a_particular_platform//:pkg", - "//_config:is_cp39_py3_none_any": "@filename_repo//:pkg", - "//_config:is_cp39_py3_none_any_linux_x86_64": "@filename_repo_for_platform//:pkg", "//conditions:default": "_no_matching_repository", }, } @@ -199,8 +185,6 @@ def _test_multiplatform_whl_aliases(env): configuration settings: //:my_config_setting //_config:is_cp39_linux_x86_64 - //_config:is_cp39_py3_none_any - //_config:is_cp39_py3_none_any_linux_x86_64 """) @@ -298,102 +282,6 @@ def _test_multiplatform_whl_aliases_nofilename_target_platforms(env): _tests.append(_test_multiplatform_whl_aliases_nofilename_target_platforms) -def _test_multiplatform_whl_aliases_filename(env): - aliases = { - whl_config_setting( - filename = "foo-0.0.3-py3-none-any.whl", - version = "3.2", - ): "foo-py3-0.0.3", - whl_config_setting( - filename = "foo-0.0.1-py3-none-any.whl", - version = "3.1", - ): "foo-py3-0.0.1", - whl_config_setting( - filename = "foo-0.0.1-cp313-cp313-any.whl", - version = "3.13", - ): "foo-cp-0.0.1", - whl_config_setting( - filename = "foo-0.0.1-cp313-cp313t-any.whl", - version = "3.13", - ): "foo-cpt-0.0.1", - whl_config_setting( - filename = "foo-0.0.2-py3-none-any.whl", - version = "3.1", - target_platforms = [ - "cp31_linux_x86_64", - "cp31_linux_aarch64", - ], - ): "foo-0.0.2", - } - got = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = [], - muslc_versions = [], - osx_versions = [], - ) - want = { - "//_config:is_cp313_cp313_any": "foo-cp-0.0.1", - "//_config:is_cp313_cp313t_any": "foo-cpt-0.0.1", - "//_config:is_cp31_py3_none_any": "foo-py3-0.0.1", - "//_config:is_cp31_py3_none_any_linux_aarch64": "foo-0.0.2", - "//_config:is_cp31_py3_none_any_linux_x86_64": "foo-0.0.2", - "//_config:is_cp32_py3_none_any": "foo-py3-0.0.3", - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_multiplatform_whl_aliases_filename) - -def _test_multiplatform_whl_aliases_filename_versioned(env): - aliases = { - whl_config_setting( - filename = "foo-0.0.1-py3-none-manylinux_2_17_x86_64.whl", - version = "3.1", - ): "glibc-2.17", - whl_config_setting( - filename = "foo-0.0.1-py3-none-manylinux_2_18_x86_64.whl", - version = "3.1", - ): "glibc-2.18", - whl_config_setting( - filename = "foo-0.0.1-py3-none-musllinux_1_1_x86_64.whl", - version = "3.1", - ): "musl-1.1", - } - got = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = [(2, 17), (2, 18)], - muslc_versions = [(1, 1), (1, 2)], - osx_versions = [], - ) - want = { - # This could just work with: - # select({ - # "//_config:is_gt_eq_2.18": "//_config:is_cp3.1_py3_none_manylinux_x86_64", - # "//conditions:default": "//_config:is_gt_eq_2.18", - # }): "glibc-2.18", - # select({ - # "//_config:is_range_2.17_2.18": "//_config:is_cp3.1_py3_none_manylinux_x86_64", - # "//_config:is_glibc_default": "//_config:is_cp3.1_py3_none_manylinux_x86_64", - # "//conditions:default": "//_config:is_glibc_default", - # }): "glibc-2.17", - # ( - # "//_config:is_gt_musl_1.1": "musl-1.1", - # "//_config:is_musl_default": "musl-1.1", - # ): "musl-1.1", - # - # For this to fully work we need to have the pypi:config_settings.bzl to generate the - # extra targets that use the FeatureFlagInfo and this to generate extra aliases for the - # config settings. - "//_config:is_cp31_py3_none_manylinux_2_17_x86_64": "glibc-2.17", - "//_config:is_cp31_py3_none_manylinux_2_18_x86_64": "glibc-2.18", - "//_config:is_cp31_py3_none_manylinux_x86_64": "glibc-2.17", - "//_config:is_cp31_py3_none_musllinux_1_1_x86_64": "musl-1.1", - "//_config:is_cp31_py3_none_musllinux_1_2_x86_64": "musl-1.1", - "//_config:is_cp31_py3_none_musllinux_x86_64": "musl-1.1", - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_multiplatform_whl_aliases_filename_versioned) - def _mock_alias(container): return lambda name, **kwargs: container.append(name) @@ -451,86 +339,6 @@ def _test_config_settings_exist_legacy(env): _tests.append(_test_config_settings_exist_legacy) -def _test_config_settings_exist(env): - for py_tag in ["py2.py3", "py3", "py311", "cp311"]: - if py_tag == "py2.py3": - abis = ["none"] - elif py_tag.startswith("py"): - abis = ["none", "abi3"] - else: - abis = ["none", "abi3", "cp311"] - - for abi_tag in abis: - for platform_tag, kwargs in { - "any": {}, - "macosx_11_0_arm64": { - "osx_versions": [(11, 0)], - "platform_config_settings": { - "osx_aarch64": [ - "@platforms//cpu:aarch64", - "@platforms//os:osx", - ], - }, - }, - "manylinux_2_17_x86_64": { - "glibc_versions": [(2, 17), (2, 18)], - "platform_config_settings": { - "linux_x86_64": [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - }, - }, - "manylinux_2_18_x86_64": { - "glibc_versions": [(2, 17), (2, 18)], - "platform_config_settings": { - "linux_x86_64": [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - }, - }, - "musllinux_1_1_aarch64": { - "muslc_versions": [(1, 2), (1, 1), (1, 0)], - "platform_config_settings": { - "linux_aarch64": [ - "@platforms//cpu:aarch64", - "@platforms//os:linux", - ], - }, - }, - }.items(): - aliases = { - whl_config_setting( - filename = "foo-0.0.1-{}-{}-{}.whl".format(py_tag, abi_tag, platform_tag), - version = "3.11", - ): "repo", - } - available_config_settings = [] - config_settings( - python_versions = ["3.11"], - native = struct( - alias = _mock_alias(available_config_settings), - config_setting = _mock_config_setting([]), - ), - selects = struct( - config_setting_group = _mock_config_setting_group(available_config_settings), - ), - **kwargs - ) - - got_aliases = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = kwargs.get("glibc_versions", []), - muslc_versions = kwargs.get("muslc_versions", []), - osx_versions = kwargs.get("osx_versions", []), - ) - got = [a.partition(":")[-1] for a in got_aliases] - - env.expect.that_collection(available_config_settings).contains_at_least(got) - -_tests.append(_test_config_settings_exist) - def pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index ad7f36aed6..3c6227cfdb 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -15,10 +15,6 @@ """render_pkg_aliases tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load( - "//python/private/pypi:pkg_aliases.bzl", - "get_filename_config_settings", -) # buildifier: disable=bzl-visibility load( "//python/private/pypi:render_pkg_aliases.bzl", "get_whl_flag_versions", @@ -79,17 +75,6 @@ def _test_bzlmod_aliases(env): "cp32_linux_x86_64", ], ): "pypi_32_bar_baz_linux_x86_64", - whl_config_setting( - version = "3.2", - filename = "foo-0.0.0-py3-none-any.whl", - ): "filename_repo", - whl_config_setting( - version = "3.2.2", - filename = "foo-0.0.0-py3-none-any.whl", - target_platforms = [ - "cp32.2_linux_x86_64", - ], - ): "filename_repo_linux_x86_64", }, }, extra_hub_aliases = {"bar_baz": ["foo"]}, @@ -117,15 +102,6 @@ pkg_aliases( config_setting = "//:my_config_setting", version = "3.2", ): "pypi_32_bar_baz_linux_x86_64", - whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", - version = "3.2", - ): "filename_repo", - whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", - target_platforms = ("cp32_linux_x86_64",), - version = "3.2.2", - ): "filename_repo_linux_x86_64", }, extra_aliases = ["foo"], )""" @@ -256,252 +232,6 @@ def _test_get_python_versions_with_target_platforms(env): _tests.append(_test_get_python_versions_with_target_platforms) -def _test_get_python_versions_from_filenames(env): - got = get_whl_flag_versions( - settings = [ - whl_config_setting( - version = "3.3", - filename = "foo-0.0.0-py3-none-" + plat + ".whl", - ) - for plat in [ - "linux_x86_64", - "manylinux_2_17_x86_64", - "manylinux_2_14_aarch64.musllinux_1_1_aarch64", - "musllinux_1_0_x86_64", - "manylinux2014_x86_64.manylinux_2_17_x86_64", - "macosx_11_0_arm64", - "macosx_10_9_x86_64", - "macosx_10_9_universal2", - "windows_x86_64", - ] - ], - ) - want = { - "glibc_versions": [(2, 14), (2, 17)], - "muslc_versions": [(1, 0), (1, 1)], - "osx_versions": [(10, 9), (11, 0)], - "python_versions": ["3.3"], - "target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "osx_x86_64", - "windows_x86_64", - ], - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_get_python_versions_from_filenames) - -def _test_get_flag_versions_from_alias_target_platforms(env): - got = get_whl_flag_versions( - settings = [ - whl_config_setting( - version = "3.3", - filename = "foo-0.0.0-py3-none-" + plat + ".whl", - ) - for plat in [ - "windows_x86_64", - ] - ] + [ - whl_config_setting( - version = "3.3", - filename = "foo-0.0.0-py3-none-any.whl", - target_platforms = [ - "cp33_linux_x86_64", - ], - ), - ], - ) - want = { - "python_versions": ["3.3"], - "target_platforms": [ - "linux_x86_64", - "windows_x86_64", - ], - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_get_flag_versions_from_alias_target_platforms) - -def _test_config_settings( - env, - *, - filename, - want, - python_version, - want_versions = {}, - target_platforms = [], - glibc_versions = [], - muslc_versions = [], - osx_versions = []): - got, got_default_version_settings = get_filename_config_settings( - filename = filename, - target_platforms = target_platforms, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, - python_version = python_version, - ) - env.expect.that_collection(got).contains_exactly(want) - env.expect.that_dict(got_default_version_settings).contains_exactly(want_versions) - -def _test_sdist(env): - # Do the first test for multiple extensions - for ext in [".tar.gz", ".zip"]: - _test_config_settings( - env, - filename = "foo-0.0.1" + ext, - python_version = "3.2", - want = [":is_cp32_sdist"], - ) - - ext = ".zip" - _test_config_settings( - env, - filename = "foo-0.0.1" + ext, - python_version = "3.2", - target_platforms = [ - "linux_aarch64", - "linux_x86_64", - ], - want = [ - ":is_cp32_sdist_linux_aarch64", - ":is_cp32_sdist_linux_x86_64", - ], - ) - -_tests.append(_test_sdist) - -def _test_py2_py3_none_any(env): - _test_config_settings( - env, - filename = "foo-0.0.1-py2.py3-none-any.whl", - python_version = "3.2", - want = [ - ":is_cp32_py_none_any", - ], - ) - - _test_config_settings( - env, - filename = "foo-0.0.1-py2.py3-none-any.whl", - python_version = "3.2", - target_platforms = [ - "osx_x86_64", - ], - want = [":is_cp32_py_none_any_osx_x86_64"], - ) - -_tests.append(_test_py2_py3_none_any) - -def _test_py3_none_any(env): - _test_config_settings( - env, - filename = "foo-0.0.1-py3-none-any.whl", - python_version = "3.1", - want = [":is_cp31_py3_none_any"], - ) - - _test_config_settings( - env, - filename = "foo-0.0.1-py3-none-any.whl", - python_version = "3.1", - target_platforms = ["linux_x86_64"], - want = [":is_cp31_py3_none_any_linux_x86_64"], - ) - -_tests.append(_test_py3_none_any) - -def _test_py3_none_macosx_10_9_universal2(env): - _test_config_settings( - env, - filename = "foo-0.0.1-py3-none-macosx_10_9_universal2.whl", - python_version = "3.1", - osx_versions = [ - (10, 9), - (11, 0), - ], - want = [], - want_versions = { - ":is_cp31_py3_none_osx_universal2": { - (10, 9): ":is_cp31_py3_none_osx_10_9_universal2", - (11, 0): ":is_cp31_py3_none_osx_11_0_universal2", - }, - }, - ) - -_tests.append(_test_py3_none_macosx_10_9_universal2) - -def _test_cp37_abi3_linux_x86_64(env): - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-abi3-linux_x86_64.whl", - python_version = "3.7", - want = [":is_cp37_abi3_linux_x86_64"], - ) - -_tests.append(_test_cp37_abi3_linux_x86_64) - -def _test_cp37_abi3_windows_x86_64(env): - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-abi3-windows_x86_64.whl", - python_version = "3.7", - want = [":is_cp37_abi3_windows_x86_64"], - ) - -_tests.append(_test_cp37_abi3_windows_x86_64) - -def _test_cp37_abi3_manylinux_2_17_x86_64(env): - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", - python_version = "3.7", - glibc_versions = [ - (2, 16), - (2, 17), - (2, 18), - ], - want = [], - want_versions = { - ":is_cp37_abi3_manylinux_x86_64": { - (2, 17): ":is_cp37_abi3_manylinux_2_17_x86_64", - (2, 18): ":is_cp37_abi3_manylinux_2_18_x86_64", - }, - }, - ) - -_tests.append(_test_cp37_abi3_manylinux_2_17_x86_64) - -def _test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64(env): - # I've seen such a wheel being built for `uv` - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-cp37-manylinux_2_17_arm64.musllinux_1_1_arm64.whl", - python_version = "3.7", - glibc_versions = [ - (2, 16), - (2, 17), - (2, 18), - ], - muslc_versions = [ - (1, 1), - ], - want = [], - want_versions = { - ":is_cp37_cp37_manylinux_aarch64": { - (2, 17): ":is_cp37_cp37_manylinux_2_17_aarch64", - (2, 18): ":is_cp37_cp37_manylinux_2_18_aarch64", - }, - ":is_cp37_cp37_musllinux_aarch64": { - (1, 1): ":is_cp37_cp37_musllinux_1_1_aarch64", - }, - }, - ) - -_tests.append(_test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64) - def render_pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/pypi/select_whl/BUILD.bazel b/tests/pypi/select_whl/BUILD.bazel new file mode 100644 index 0000000000..0ad8cba0cd --- /dev/null +++ b/tests/pypi/select_whl/BUILD.bazel @@ -0,0 +1,3 @@ +load(":select_whl_tests.bzl", "select_whl_test_suite") + +select_whl_test_suite(name = "select_whl_tests") diff --git a/tests/pypi/whl_target_platforms/select_whl_tests.bzl b/tests/pypi/select_whl/select_whl_tests.bzl similarity index 55% rename from tests/pypi/whl_target_platforms/select_whl_tests.bzl rename to tests/pypi/select_whl/select_whl_tests.bzl index 1674ac5ef2..06da2ab8ae 100644 --- a/tests/pypi/whl_target_platforms/select_whl_tests.bzl +++ b/tests/pypi/select_whl/select_whl_tests.bzl @@ -1,22 +1,8 @@ -# Copyright 2024 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - "" load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "REPO_VERBOSITY_ENV_VAR", "repo_utils") # buildifier: disable=bzl-visibility -load("//python/private/pypi:whl_target_platforms.bzl", "select_whls") # buildifier: disable=bzl-visibility +load("//python/private/pypi:select_whl.bzl", "select_whl") # buildifier: disable=bzl-visibility WHL_LIST = [ "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", @@ -70,6 +56,9 @@ WHL_LIST = [ "pkg-0.0.1-py310-abi3-any.whl", "pkg-0.0.1-py3-abi3-any.whl", "pkg-0.0.1-py3-none-any.whl", + # Extra examples that should be discarded + "pkg-0.0.1-py27-cp27mu-win_amd64.whl", + "pkg-0.0.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", ] def _match(env, got, *want_filenames): @@ -77,15 +66,16 @@ def _match(env, got, *want_filenames): env.expect.that_collection(got).has_size(len(want_filenames)) return + got = [g for g in got if g] got_filenames = [g.filename for g in got] - env.expect.that_collection(got_filenames).contains_exactly(want_filenames) + env.expect.that_collection(got_filenames).contains_exactly(want_filenames).in_order() if got: # Check that we pass the original structs env.expect.that_str(got[0].other).equals("dummy") -def _select_whls(whls, debug = False, **kwargs): - return select_whls( +def _select_whl(whls, debug = False, **kwargs): + return select_whl( whls = [ struct( filename = f, @@ -106,204 +96,301 @@ def _select_whls(whls, debug = False, **kwargs): _tests = [] -def _test_simplest(env): - got = _select_whls( +def _test_not_select_py2(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = [ + "pkg-0.0.1-py2-none-any.whl", + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + ], + platforms = ["any"], + whl_abi_tags = ["none"], + python_version = "3.13", + limit = 2, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + ) + +_tests.append(_test_not_select_py2) + +def _test_not_select_abi3(env): + # Check we prefer platform specific wheels + got = _select_whl( whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-none-any.whl", + # the following should be ignored "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-p1.p2.p2.whl", + ], + platforms = ["any", "p1"], + whl_abi_tags = ["none"], + python_version = "3.13", + limit = 2, + debug = True, + ) + _match( + env, + got, + "pkg-0.0.1-py3-none-any.whl", + ) + +_tests.append(_test_not_select_abi3) + +def _test_select_cp312(env): + # Check we prefer platform specific wheels + got = _select_whl( + whls = [ + "pkg-0.0.1-py2-none-any.whl", "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + "pkg-0.0.1-cp39-none-any.whl", + "pkg-0.0.1-cp312-none-any.whl", + "pkg-0.0.1-cp314-none-any.whl", ], - want_platforms = ["cp30_ignored"], + platforms = ["any"], + whl_abi_tags = ["none"], + python_version = "3.13", + limit = 5, ) _match( env, got, + "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py312-none-any.whl", + "pkg-0.0.1-cp39-none-any.whl", + "pkg-0.0.1-cp312-none-any.whl", + ) + +_tests.append(_test_select_cp312) + +def _test_simplest(env): + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", "pkg-0.0.1-py3-abi3-any.whl", "pkg-0.0.1-py3-none-any.whl", + ] + + got = _select_whl( + whls = whls, + platforms = ["any"], + whl_abi_tags = ["abi3"], + python_version = "3.0", + ) + _match( + env, + [got], + "pkg-0.0.1-py3-abi3-any.whl", ) _tests.append(_test_simplest) def _test_select_by_supported_py_version(env): + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py311-abi3-any.whl", + ] + for minor_version, match in { 8: "pkg-0.0.1-py3-abi3-any.whl", 11: "pkg-0.0.1-py311-abi3-any.whl", }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py311-abi3-any.whl", - ], - want_platforms = ["cp3{}_ignored".format(minor_version)], + got = _select_whl( + whls = whls, + platforms = ["any"], + whl_abi_tags = ["abi3"], + python_version = "3.{}".format(minor_version), ) - _match(env, got, match) + _match(env, [got], match) _tests.append(_test_select_by_supported_py_version) def _test_select_by_supported_cp_version(env): + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py311-abi3-any.whl", + "pkg-0.0.1-cp311-abi3-any.whl", + ] + for minor_version, match in { 11: "pkg-0.0.1-cp311-abi3-any.whl", 8: "pkg-0.0.1-py3-abi3-any.whl", }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py311-abi3-any.whl", - "pkg-0.0.1-cp311-abi3-any.whl", - ], - want_platforms = ["cp3{}_ignored".format(minor_version)], + got = _select_whl( + whls = whls, + platforms = ["any"], + whl_abi_tags = ["abi3"], + python_version = "3.{}".format(minor_version), ) - _match(env, got, match) + _match(env, [got], match) _tests.append(_test_select_by_supported_cp_version) def _test_supported_cp_version_manylinux(env): + whls = [ + "pkg-0.0.1-py2.py3-none-manylinux_x86_64.whl", + "pkg-0.0.1-py3-none-manylinux_x86_64.whl", + "pkg-0.0.1-py311-none-manylinux_x86_64.whl", + "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", + ] + for minor_version, match in { 8: "pkg-0.0.1-py3-none-manylinux_x86_64.whl", 11: "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", }.items(): - got = _select_whls( - whls = [ - "pkg-0.0.1-py2.py3-none-manylinux_x86_64.whl", - "pkg-0.0.1-py3-none-manylinux_x86_64.whl", - "pkg-0.0.1-py311-none-manylinux_x86_64.whl", - "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", - ], - want_platforms = ["cp3{}_linux_x86_64".format(minor_version)], + got = _select_whl( + whls = whls, + platforms = ["manylinux_x86_64"], + whl_abi_tags = ["none"], + python_version = "3.{}".format(minor_version), ) - _match(env, got, match) + _match(env, [got], match) _tests.append(_test_supported_cp_version_manylinux) def _test_ignore_unsupported(env): - got = _select_whls( - whls = [ - "pkg-0.0.1-xx3-abi3-any.whl", - ], - want_platforms = ["cp30_ignored"], + whls = ["pkg-0.0.1-xx3-abi3-any.whl"] + got = _select_whl( + whls = whls, + platforms = ["any"], + whl_abi_tags = ["none"], + python_version = "3.0", ) - _match(env, got) + if got: + _match(env, [got], None) _tests.append(_test_ignore_unsupported) def _test_match_abi_and_not_py_version(env): # Check we match the ABI and not the py version - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp37_linux_x86_64"]) + whls = WHL_LIST + platforms = [ + "musllinux_*_x86_64", + "manylinux_*_x86_64", + ] + got = _select_whl( + whls = whls, + platforms = platforms, + whl_abi_tags = ["abi3", "cp37m"], + python_version = "3.7", + ) _match( env, - got, + [got], "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + ) + + got = _select_whl( + whls = whls, + platforms = platforms[::-1], + whl_abi_tags = ["abi3", "cp37m"], + python_version = "3.7", + ) + _match( + env, + [got], "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", ) _tests.append(_test_match_abi_and_not_py_version) def _test_select_filename_with_many_tags(env): # Check we can select a filename with many platform tags - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp39_linux_x86_32"]) + got = _select_whl( + whls = WHL_LIST, + platforms = [ + "any", + "musllinux_*_i686", + "manylinux_*_i686", + ], + whl_abi_tags = ["none", "abi3", "cp39"], + python_version = "3.9", + limit = 5, + ) _match( env, got, - "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp39-abi3-any.whl", "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-cp39-abi3-any.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", ) _tests.append(_test_select_filename_with_many_tags) -def _test_osx_prefer_arch_specific(env): - # Check that we prefer the specific wheel - got = _select_whls( +def _test_freethreaded_wheels(env): + # Check we prefer platform specific wheels + got = _select_whl( whls = WHL_LIST, - want_platforms = ["cp311_osx_x86_64", "cp311_osx_x86_32"], + platforms = [ + "any", + "musllinux_*_x86_64", + ], + whl_abi_tags = ["none", "abi3", "cp313", "cp313t"], + python_version = "3.13", + limit = 8, ) _match( env, got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", + # The last item has the most priority "pkg-0.0.1-py3-none-any.whl", - ) - - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp311_osx_aarch64"]) - _match( - env, - got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py310-abi3-any.whl", "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", + "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", ) -_tests.append(_test_osx_prefer_arch_specific) +_tests.append(_test_freethreaded_wheels) -def _test_osx_fallback_to_universal2(env): - # Check that we can use the universal2 if the arm wheel is not available - got = _select_whls( - whls = [w for w in WHL_LIST if "arm64" not in w], - want_platforms = ["cp311_osx_aarch64"], +def _test_pytags_all_possible(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313-none-win_amd64.whl", + ], + platforms = ["win_amd64"], + whl_abi_tags = ["none"], + python_version = "3.12", ) _match( env, - got, - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", + [got], + "pkg-0.0.1-py2.py27.py3.py30.py31.py32.py33.py34.py35.py36.py37.py38.py39.py310.py311.py312.py313-none-win_amd64.whl", ) -_tests.append(_test_osx_fallback_to_universal2) +_tests.append(_test_pytags_all_possible) -def _test_prefer_manylinux_wheels(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp39_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - ) - -_tests.append(_test_prefer_manylinux_wheels) - -def _test_freethreaded_wheels(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313_linux_x86_64"]) - _match( - env, - got, - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", +def _test_manylinx_musllinux_pref(env): + got = _select_whl( + whls = [ + "pkg-0.0.1-py3-none-manylinux_2_31_x86_64.musllinux_1_1_x86_64.whl", + ], + platforms = [ + "manylinux_*_x86_64", + "musllinux_*_x86_64", + ], + whl_abi_tags = ["none"], + python_version = "3.12", + limit = 2, ) - -_tests.append(_test_freethreaded_wheels) - -def _test_micro_version_freethreaded(env): - # Check we prefer platform specific wheels - got = _select_whls(whls = WHL_LIST, want_platforms = ["cp313.3_linux_x86_64"]) _match( env, got, - "pkg-0.0.1-cp313-cp313t-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-cp313-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-abi3-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp313-none-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", + # there is only wheel, just select that + "pkg-0.0.1-py3-none-manylinux_2_31_x86_64.musllinux_1_1_x86_64.whl", ) -_tests.append(_test_micro_version_freethreaded) +_tests.append(_test_manylinx_musllinux_pref) def select_whl_test_suite(name): """Create the test suite. diff --git a/tests/pypi/whl_target_platforms/BUILD.bazel b/tests/pypi/whl_target_platforms/BUILD.bazel index 6c35b08d32..fec25af033 100644 --- a/tests/pypi/whl_target_platforms/BUILD.bazel +++ b/tests/pypi/whl_target_platforms/BUILD.bazel @@ -12,9 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load(":select_whl_tests.bzl", "select_whl_test_suite") load(":whl_target_platforms_tests.bzl", "whl_target_platforms_test_suite") -select_whl_test_suite(name = "select_whl_tests") - whl_target_platforms_test_suite(name = "whl_target_platforms_tests")