diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel index 847c85d634..cb3408a191 100644 --- a/python/private/pypi/BUILD.bazel +++ b/python/private/pypi/BUILD.bazel @@ -320,8 +320,6 @@ bzl_library( srcs = ["pkg_aliases.bzl"], deps = [ ":labels_bzl", - ":parse_whl_name_bzl", - ":whl_target_platforms_bzl", "//python/private:text_util_bzl", "@bazel_skylib//lib:selects", ], @@ -349,9 +347,7 @@ bzl_library( srcs = ["render_pkg_aliases.bzl"], deps = [ ":generate_group_library_build_bazel_bzl", - ":parse_whl_name_bzl", ":whl_config_setting_bzl", - ":whl_target_platforms_bzl", "//python/private:normalize_name_bzl", "//python/private:text_util_bzl", ], diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl index f4826007f8..dcb6779d5b 100644 --- a/python/private/pypi/config_settings.bzl +++ b/python/private/pypi/config_settings.bzl @@ -17,101 +17,17 @@ The {obj}`config_settings` macro is used to create the config setting targets that can be used in the {obj}`pkg_aliases` macro for selecting the compatible repositories. -Bazel's selects work by selecting the most-specialized configuration setting -that matches the target platform, which is further described in [bazel documentation][docs]. -We can leverage this fact to ensure that the most specialized matches are used -by default with the users being able to configure string_flag values to select -the less specialized ones. - -[docs]: https://bazel.build/docs/configurable-attributes - -The config settings in the order from the least specialized to the most -specialized is as follows: -* `:is_cp3` -* `:is_cp3_sdist` -* `:is_cp3_py_none_any` -* `:is_cp3_py3_none_any` -* `:is_cp3_py3_abi3_any` -* `:is_cp3_none_any` -* `:is_cp3_any_any` -* `:is_cp3_cp3_any` and `:is_cp3_cp3t_any` -* `:is_cp3_py_none_` -* `:is_cp3_py3_none_` -* `:is_cp3_py3_abi3_` -* `:is_cp3_none_` -* `:is_cp3_abi3_` -* `:is_cp3_cp3_` and `:is_cp3_cp3t_` - -Optionally instead of `` there sometimes may be `.` used in order to fully specify the versions - -The specialization of free-threaded vs non-free-threaded wheels is the same as -they are just variants of each other. The same goes for the specialization of -`musllinux` vs `manylinux`. - -The goal of this macro is to provide config settings that provide unambigous -matches if any pair of them is used together for any target configuration -setting. We achieve this by using dummy internal `flag_values` keys to force the -items further down the list to appear to be more specialized than the ones above. - -What is more, the names of the config settings are as similar to the platform wheel -specification as possible. How the wheel names map to the config setting names defined -in here is described in {obj}`pkg_aliases` documentation. - -:::{note} -Right now the specialization of adjacent config settings where one is with -`constraint_values` and one is without is ambiguous. I.e. `py_none_any` and -`sdist_linux_x86_64` have the same specialization from bazel point of view -because one has one `flag_value` entry and `constraint_values` and the -other has 2 flag_value entries. And unfortunately there is no way to disambiguate -it, because we are essentially in two dimensions here (`flag_values` and -`constraint_values`). Hence, when using the `config_settings` from here, -either have all of them with empty `suffix` or all of them with a non-empty -suffix. -::: +The config settings are of the form `:is_cp3`. Suffix is a +normalized user provided value for the platform name. The goal of this macro is to +ensure that we can incorporate the user provided `config_setting` targets to create +our composite config_setting targets. """ load("@bazel_skylib//lib:selects.bzl", "selects") -load("//python/private:flags.bzl", "LibcFlag") -load(":flags.bzl", "INTERNAL_FLAGS", "UniversalWhlFlag") - -FLAGS = struct( - **{ - f: str(Label("//python/config_settings:" + f)) - for f in [ - "is_pip_whl_auto", - "is_pip_whl_no", - "is_pip_whl_only", - "_is_py_freethreaded_yes", - "_is_py_freethreaded_no", - "pip_whl_glibc_version", - "pip_whl_muslc_version", - "pip_whl_osx_arch", - "pip_whl_osx_version", - "py_linux_libc", - "python_version", - ] - } -) - -_DEFAULT = "//conditions:default" -_INCOMPATIBLE = "@platforms//:incompatible" - -# Here we create extra string flags that are just to work with the select -# selecting the most specialized match. We don't allow the user to change -# them. -_flags = struct( - **{ - f: str(Label("//python/config_settings:_internal_pip_" + f)) - for f in INTERNAL_FLAGS - } -) def config_settings( *, python_versions = [], - glibc_versions = [], - muslc_versions = [], - osx_versions = [], name = None, platform_config_settings = {}, **kwargs): @@ -121,12 +37,6 @@ def config_settings( name (str): Currently unused. python_versions (list[str]): The list of python versions to configure config settings for. - glibc_versions (list[str]): The list of glibc version of the wheels to - configure config settings for. - muslc_versions (list[str]): The list of musl version of the wheels to - configure config settings for. - osx_versions (list[str]): The list of OSX OS versions to configure - config settings for. platform_config_settings: {type}`dict[str, list[str]]` the constraint values to use instead of the default ones. Key are platform names (a human-friendly platform string). Values are lists of @@ -134,212 +44,46 @@ def config_settings( **kwargs: Other args passed to the underlying implementations, such as {obj}`native`. """ - - glibc_versions = [""] + glibc_versions - muslc_versions = [""] + muslc_versions - osx_versions = [""] + osx_versions target_platforms = { "": [], - # TODO @aignas 2025-06-15: allowing universal2 and platform specific wheels in one - # closure is making things maybe a little bit too complicated. - "osx_universal2": ["@platforms//os:osx"], } | platform_config_settings - for python_version in python_versions: - for platform_name, config_settings in target_platforms.items(): - suffix = "_{}".format(platform_name) if platform_name else "" - os, _, cpu = platform_name.partition("_") + for platform_name, config_settings in target_platforms.items(): + suffix = "_{}".format(platform_name) if platform_name else "" + + # We parse the target settings and if there is a "platforms//os" or + # "platforms//cpu" value in here, we also add it into the constraint_values + # + # this is to ensure that we can still pass all of the unit tests for config + # setting specialization. + # + # TODO @aignas 2025-07-23: is this the right way? Maybe we should drop these + # and remove the tests? + constraint_values = [] + for setting in config_settings: + setting_label = Label(setting) + if setting_label.repo_name == "platforms" and setting_label.package in ["os", "cpu"]: + constraint_values.append(setting) + + for python_version in python_versions: + cpv = "cp" + python_version.replace(".", "") + prefix = "is_{}".format(cpv) - # We parse the target settings and if there is a "platforms//os" or - # "platforms//cpu" value in here, we also add it into the constraint_values - # - # this is to ensure that we can still pass all of the unit tests for config - # setting specialization. - constraint_values = [] - for setting in config_settings: - setting_label = Label(setting) - if setting_label.repo_name == "platforms" and setting_label.package in ["os", "cpu"]: - constraint_values.append(setting) - - _dist_config_settings( - suffix = suffix, - plat_flag_values = _plat_flag_values( - os = os, - cpu = cpu, - osx_versions = osx_versions, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - ), + _dist_config_setting( + name = prefix + suffix, + flag_values = { + Label("//python/config_settings:python_version_major_minor"): python_version, + }, config_settings = config_settings, constraint_values = constraint_values, - python_version = python_version, **kwargs ) -def _dist_config_settings(*, suffix, plat_flag_values, python_version, **kwargs): - flag_values = { - Label("//python/config_settings:python_version_major_minor"): python_version, - } - - cpv = "cp" + python_version.replace(".", "") - prefix = "is_{}".format(cpv) - - _dist_config_setting( - name = prefix + suffix, - flag_values = flag_values, - **kwargs - ) - - flag_values[_flags.dist] = "" - - # First create an sdist, we will be building upon the flag values, which - # will ensure that each sdist config setting is the least specialized of - # all. However, we need at least one flag value to cover the case where we - # have `sdist` for any platform, hence we have a non-empty `flag_values` - # here. - _dist_config_setting( - name = "{}_sdist{}".format(prefix, suffix), - flag_values = flag_values, - compatible_with = (FLAGS.is_pip_whl_no, FLAGS.is_pip_whl_auto), - **kwargs - ) - - used_flags = {} - - # NOTE @aignas 2024-12-01: the abi3 is not compatible with freethreaded - # builds as per PEP703 (https://peps.python.org/pep-0703/#backwards-compatibility) - # - # The discussion here also reinforces this notion: - # https://discuss.python.org/t/pep-703-making-the-global-interpreter-lock-optional-3-12-updates/26503/99 - - for name, f, compatible_with in [ - ("py_none", _flags.whl, None), - ("py3_none", _flags.whl_py3, None), - ("py3_abi3", _flags.whl_py3_abi3, (FLAGS._is_py_freethreaded_no,)), - ("none", _flags.whl_pycp3x, None), - ("abi3", _flags.whl_pycp3x_abi3, (FLAGS._is_py_freethreaded_no,)), - # The below are not specializations of one another, they are variants - (cpv, _flags.whl_pycp3x_abicp, (FLAGS._is_py_freethreaded_no,)), - (cpv + "t", _flags.whl_pycp3x_abicp, (FLAGS._is_py_freethreaded_yes,)), - ]: - if (f, compatible_with) in used_flags: - # This should never happen as all of the different whls should have - # unique flag values - fail("BUG: the flag {} is attempted to be added twice to the list".format(f)) - else: - flag_values[f] = "yes" if f == _flags.whl else "" - used_flags[(f, compatible_with)] = True - - _dist_config_setting( - name = "{}_{}_any{}".format(prefix, name, suffix), - flag_values = flag_values, - compatible_with = compatible_with, - **kwargs - ) - - generic_flag_values = flag_values - generic_used_flags = used_flags - - for (suffix, flag_values) in plat_flag_values: - used_flags = {(f, None): True for f in flag_values} | generic_used_flags - flag_values = flag_values | generic_flag_values - - for name, f, compatible_with in [ - ("py_none", _flags.whl_plat, None), - ("py3_none", _flags.whl_plat_py3, None), - ("py3_abi3", _flags.whl_plat_py3_abi3, (FLAGS._is_py_freethreaded_no,)), - ("none", _flags.whl_plat_pycp3x, None), - ("abi3", _flags.whl_plat_pycp3x_abi3, (FLAGS._is_py_freethreaded_no,)), - # The below are not specializations of one another, they are variants - (cpv, _flags.whl_plat_pycp3x_abicp, (FLAGS._is_py_freethreaded_no,)), - (cpv + "t", _flags.whl_plat_pycp3x_abicp, (FLAGS._is_py_freethreaded_yes,)), - ]: - if (f, compatible_with) in used_flags: - # This should never happen as all of the different whls should have - # unique flag values. - fail("BUG: the flag {} is attempted to be added twice to the list".format(f)) - else: - flag_values[f] = "" - used_flags[(f, compatible_with)] = True - - _dist_config_setting( - name = "{}_{}_{}".format(prefix, name, suffix), - flag_values = flag_values, - compatible_with = compatible_with, - **kwargs - ) - -def _to_version_string(version, sep = "."): - if not version: - return "" - - return "{}{}{}".format(version[0], sep, version[1]) - -def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions): - ret = [] - if os == "": - return [] - elif os == "windows": - ret.append(("{}_{}".format(os, cpu), {})) - elif os == "osx": - for osx_version in osx_versions: - flags = { - FLAGS.pip_whl_osx_version: _to_version_string(osx_version), - } - if cpu != "universal2": - flags[FLAGS.pip_whl_osx_arch] = UniversalWhlFlag.ARCH - - if not osx_version: - suffix = "{}_{}".format(os, cpu) - else: - suffix = "{}_{}_{}".format(os, _to_version_string(osx_version, "_"), cpu) - - ret.append((suffix, flags)) - - elif os == "linux": - for os_prefix, linux_libc in { - os: LibcFlag.GLIBC, - "many" + os: LibcFlag.GLIBC, - "musl" + os: LibcFlag.MUSL, - }.items(): - if linux_libc == LibcFlag.GLIBC: - libc_versions = glibc_versions - libc_flag = FLAGS.pip_whl_glibc_version - elif linux_libc == LibcFlag.MUSL: - libc_versions = muslc_versions - libc_flag = FLAGS.pip_whl_muslc_version - else: - fail("Unsupported libc type: {}".format(linux_libc)) - - for libc_version in libc_versions: - if libc_version and os_prefix == os: - continue - elif libc_version: - suffix = "{}_{}_{}".format(os_prefix, _to_version_string(libc_version, "_"), cpu) - else: - suffix = "{}_{}".format(os_prefix, cpu) - - ret.append(( - suffix, - { - FLAGS.py_linux_libc: linux_libc, - libc_flag: _to_version_string(libc_version), - }, - )) - else: - fail("Unsupported os: {}".format(os)) - - return ret - -def _dist_config_setting(*, name, compatible_with = None, selects = selects, native = native, config_settings = None, **kwargs): +def _dist_config_setting(*, name, selects = selects, native = native, config_settings = None, **kwargs): """A macro to create a target for matching Python binary and source distributions. Args: name: The name of the public target. - compatible_with: {type}`tuple[Label]` A collection of config settings that are - compatible with the given dist config setting. For example, if only - non-freethreaded python builds are allowed, add - FLAGS._is_py_freethreaded_no here. config_settings: {type}`list[str | Label]` the list of target settings that must be matched before we try to evaluate the config_setting that we may create in this function. @@ -352,18 +96,6 @@ def _dist_config_setting(*, name, compatible_with = None, selects = selects, nat **kwargs: The kwargs passed to the config_setting rule. Visibility of the main alias target is also taken from the kwargs. """ - if compatible_with: - dist_config_setting_name = "_" + name - native.alias( - name = name, - actual = select( - {setting: dist_config_setting_name for setting in compatible_with} | { - _DEFAULT: _INCOMPATIBLE, - }, - ), - visibility = kwargs.get("visibility"), - ) - name = dist_config_setting_name # first define the config setting that has all of the constraint values _name = "_" + name diff --git a/python/private/pypi/hub_repository.bzl b/python/private/pypi/hub_repository.bzl index 75f3ec98d7..1d572d09e2 100644 --- a/python/private/pypi/hub_repository.bzl +++ b/python/private/pypi/hub_repository.bzl @@ -142,10 +142,6 @@ def whl_config_settings_to_json(repo_mapping): def _whl_config_setting_dict(a): ret = {} - if a.config_setting: - ret["config_setting"] = a.config_setting - if a.filename: - ret["filename"] = a.filename if a.target_platforms: ret["target_platforms"] = a.target_platforms if a.version: diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl index 4d3cc61590..67ce297466 100644 --- a/python/private/pypi/pkg_aliases.bzl +++ b/python/private/pypi/pkg_aliases.bzl @@ -23,54 +23,12 @@ Definitions: :suffix: Can be either empty or `__`, which is usually used to distinguish multiple versions used for different target platforms. :os: OS identifier that exists in `@platforms//os:`. :cpu: CPU architecture identifier that exists in `@platforms//cpu:`. -:python_tag: The Python tag as defined by the [Python Packaging Authority][packaging_spec]. E.g. `py2.py3`, `py3`, `py311`, `cp311`. -:abi_tag: The ABI tag as defined by the [Python Packaging Authority][packaging_spec]. E.g. `none`, `abi3`, `cp311`, `cp311t`. -:platform_tag: The Platform tag as defined by the [Python Packaging Authority][packaging_spec]. E.g. `manylinux_2_17_x86_64`. -:platform_suffix: is a derivative of the `platform_tag` and is used to implement selection based on `libc` or `osx` version. All of the config settings used by this macro are generated by {obj}`config_settings`, for more detailed documentation on what each config setting maps to and their precedence, refer to documentation on that page. -The first group of config settings that are as follows: - -* `//_config:is_cp3` is used to select legacy `pip` - based `whl` and `sdist` {obj}`whl_library` instances. Whereas other config - settings are created when {obj}`pip.parse.experimental_index_url` is used. -* `//_config:is_cp3_sdist` is for wheels built from - `sdist` in {obj}`whl_library`. -* `//_config:is_cp3_py__any` for wheels with - `py2.py3` `python_tag` value. -* `//_config:is_cp3_py3__any` for wheels with - `py3` `python_tag` value. -* `//_config:is_cp3__any` for any other wheels. -* `//_config:is_cp3_py__` for - platform-specific wheels with `py2.py3` `python_tag` value. -* `//_config:is_cp3_py3__` for - platform-specific wheels with `py3` `python_tag` value. -* `//_config:is_cp3__` for any other - platform-specific wheels. - -Note that wheels with `abi3` or `none` `abi_tag` values and `python_tag` values -other than `py2.py3` or `py3` are compatible with the python version that is -equal or higher than the one denoted in the `python_tag`. For example: `py37` -and `cp37` wheels are compatible with Python 3.7 and above and in the case of -the target python version being `3.11`, `rules_python` will use -`//_config:is_cp311__any` config settings. - -For platform-specific wheels, i.e. the ones that have their `platform_tag` as -something else than `any`, we treat them as below: -* `linux_` tags assume that the target `libc` flavour is `glibc`, so this - is in many ways equivalent to it being `manylinux`, but with an unspecified - `libc` version. -* For `osx` and `linux` OSes wheel filename will be mapped to multiple config settings: - * `osx_` and `osx___` where - `major_version` and `minor_version` are the compatible OSX versions. - * `linux_` and - `linux___` where the version - identifiers are the compatible libc versions. - -[packaging_spec]: https://packaging.python.org/en/latest/specifications/platform-compatibility-tags/ +`//_config:is_cp3` is used to select any target platforms. """ load("@bazel_skylib//lib:selects.bzl", "selects") @@ -85,14 +43,6 @@ load( "WHEEL_FILE_IMPL_LABEL", "WHEEL_FILE_PUBLIC_LABEL", ) -load(":parse_whl_name.bzl", "parse_whl_name") -load(":whl_target_platforms.bzl", "whl_target_platforms") - -# This value is used as sentinel value in the alias/config setting machinery -# for libc and osx versions. If we encounter this version in this part of the -# code, then it means that we have a bug in rules_python and that we should fix -# it. It is more of an internal consistency check. -_VERSION_NONE = (0, 0) _NO_MATCH_ERROR_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -137,7 +87,7 @@ def pkg_aliases( to bazel skylib's `selects.with_or`, so they can be tuples as well. group_name: {type}`str` The group name that the pkg belongs to. extra_aliases: {type}`list[str]` The extra aliases to be created. - **kwargs: extra kwargs to pass to {bzl:obj}`get_filename_config_settings`. + **kwargs: extra kwargs to pass to {bzl:obj}`get_config_settings`. """ alias = kwargs.pop("native", native).alias select = kwargs.pop("select", selects.with_or) @@ -219,21 +169,9 @@ def pkg_aliases( actual = "//_groups:{}_whl".format(group_name), ) -def _normalize_versions(name, versions): - if not versions: - return [] - - if _VERSION_NONE in versions: - fail("a sentinel version found in '{}', check render_pkg_aliases for bugs".format(name)) - - return sorted(versions) - def multiplatform_whl_aliases( *, - aliases = [], - glibc_versions = [], - muslc_versions = [], - osx_versions = []): + aliases = []): """convert a list of aliases from filename to config_setting ones. Exposed only for unit tests. @@ -243,12 +181,6 @@ def multiplatform_whl_aliases( to process. Any aliases that have the filename set will be converted to a dict of config settings to repo names. The struct is created by {func}`whl_config_setting`. - glibc_versions: {type}`list[tuple[int, int]]` list of versions that can be - used in this hub repo. - muslc_versions: {type}`list[tuple[int, int]]` list of versions that can be - used in this hub repo. - osx_versions: {type}`list[tuple[int, int]]` list of versions that can be - used in this hub repo. Returns: A dict with of config setting labels to repo names or the repo name itself. @@ -258,207 +190,54 @@ def multiplatform_whl_aliases( # We don't have any aliases, this is a repo name return aliases - # TODO @aignas 2024-11-17: we might be able to use FeatureFlagInfo and some - # code gen to create a version_lt_x target, which would allow us to check - # if the libc version is in a particular range. - glibc_versions = _normalize_versions("glibc_versions", glibc_versions) - muslc_versions = _normalize_versions("muslc_versions", muslc_versions) - osx_versions = _normalize_versions("osx_versions", osx_versions) - ret = {} - versioned_additions = {} for alias, repo in aliases.items(): if type(alias) != "struct": ret[alias] = repo continue - elif not (alias.filename or alias.target_platforms): - # This is an internal consistency check - fail("Expected to have either 'filename' or 'target_platforms' set, got: {}".format(alias)) - config_settings, all_versioned_settings = get_filename_config_settings( - filename = alias.filename or "", + config_settings = get_config_settings( target_platforms = alias.target_platforms, python_version = alias.version, - # If we have multiple platforms but no wheel filename, lets use different - # config settings. - non_whl_prefix = "sdist" if alias.filename else "", - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, ) for setting in config_settings: ret["//_config" + setting] = repo - # Now for the versioned platform config settings, we need to select one - # that best fits the bill and if there are multiple wheels, e.g. - # manylinux_2_17_x86_64 and manylinux_2_28_x86_64, then we need to select - # the former when the glibc is in the range of [2.17, 2.28) and then chose - # the later if it is [2.28, ...). If the 2.28 wheel was not present in - # the hub, then we would need to use 2.17 for all the glibc version - # configurations. - # - # Here we add the version settings to a dict where we key the range of - # versions that the whl spans. If the wheel supports musl and glibc at - # the same time, we do this for each supported platform, hence the - # double dict. - for default_setting, versioned in all_versioned_settings.items(): - versions = sorted(versioned) - min_version = versions[0] - max_version = versions[-1] - - versioned_additions.setdefault(default_setting, {})[(min_version, max_version)] = struct( - repo = repo, - settings = versioned, - ) - - versioned = {} - for default_setting, candidates in versioned_additions.items(): - # Sort the candidates by the range of versions the span, so that we - # start with the lowest version. - for _, candidate in sorted(candidates.items()): - # Set the default with the first candidate, which gives us the highest - # compatibility. If the users want to use a higher-version than the default - # they can configure the glibc_version flag. - versioned.setdefault("//_config" + default_setting, candidate.repo) - - # We will be overwriting previously added entries, but that is intended. - for _, setting in candidate.settings.items(): - versioned["//_config" + setting] = candidate.repo - - ret.update(versioned) return ret -def get_filename_config_settings( +def get_config_settings( *, - filename, target_platforms, - python_version, - glibc_versions = None, - muslc_versions = None, - osx_versions = None, - non_whl_prefix = "sdist"): + python_version): """Get the filename config settings. Exposed only for unit tests. Args: - filename: the distribution filename (can be a whl or an sdist). target_platforms: list[str], target platforms in "{abi}_{os}_{cpu}" format. - glibc_versions: list[tuple[int, int]], list of versions. - muslc_versions: list[tuple[int, int]], list of versions. - osx_versions: list[tuple[int, int]], list of versions. python_version: the python version to generate the config_settings for. - non_whl_prefix: the prefix of the config setting when the whl we don't have - a filename ending with ".whl". Returns: A tuple: * A list of config settings that are generated by ./pip_config_settings.bzl * The list of default version settings. """ - prefixes = [] - suffixes = [] - setting_supported_versions = {} - - if filename.endswith(".whl"): - parsed = parse_whl_name(filename) - if parsed.python_tag == "py2.py3": - py = "py_" - elif parsed.python_tag == "py3": - py = "py3_" - elif parsed.python_tag.startswith("cp"): - py = "" - else: - py = "py3_" - abi = parsed.abi_tag - - # TODO @aignas 2025-04-20: test - abi, _, _ = abi.partition(".") - - if parsed.platform_tag == "any": - prefixes = ["{}{}_any".format(py, abi)] - else: - prefixes = ["{}{}".format(py, abi)] - suffixes = _whl_config_setting_suffixes( - platform_tag = parsed.platform_tag, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, - setting_supported_versions = setting_supported_versions, - ) - else: - prefixes = [non_whl_prefix or ""] - - py = "cp{}".format(python_version).replace(".", "") prefixes = [ - "{}_{}".format(py, prefix) if prefix else py - for prefix in prefixes + "cp{}".format(python_version).replace(".", ""), ] - versioned = { - ":is_{}_{}".format(prefix, suffix): { - version: ":is_{}_{}".format(prefix, setting) - for version, setting in versions.items() - } - for prefix in prefixes - for suffix, versions in setting_supported_versions.items() - } - - if suffixes or target_platforms or versioned: + if target_platforms: target_platforms = target_platforms or [] - suffixes = suffixes or [_non_versioned_platform(p) for p in target_platforms] + suffixes = [_non_versioned_platform(p) for p in target_platforms] return [ ":is_{}_{}".format(prefix, suffix) for prefix in prefixes for suffix in suffixes - ], versioned + ] else: - return [":is_{}".format(p) for p in prefixes], setting_supported_versions - -def _whl_config_setting_suffixes( - platform_tag, - glibc_versions, - muslc_versions, - osx_versions, - setting_supported_versions): - suffixes = [] - for platform_tag in platform_tag.split("."): - for p in whl_target_platforms(platform_tag): - prefix = p.os - suffix = p.cpu - if "manylinux" in platform_tag: - prefix = "manylinux" - versions = glibc_versions - elif "musllinux" in platform_tag: - prefix = "musllinux" - versions = muslc_versions - elif p.os in ["linux", "windows"]: - versions = [(0, 0)] - elif p.os == "osx": - versions = osx_versions - if "universal2" in platform_tag: - suffix = "universal2" - else: - fail("Unsupported whl os: {}".format(p.os)) - - default_version_setting = "{}_{}".format(prefix, suffix) - supported_versions = {} - for v in versions: - if v == (0, 0): - suffixes.append(default_version_setting) - elif v >= p.version: - supported_versions[v] = "{}_{}_{}_{}".format( - prefix, - v[0], - v[1], - suffix, - ) - if supported_versions: - setting_supported_versions[default_version_setting] = supported_versions - - return suffixes + return [":is_{}".format(p) for p in prefixes] def _non_versioned_platform(p, *, strict = False): """A small utility function that converts 'cp311_linux_x86_64' to 'linux_x86_64'. diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl index e743fc20f7..0a1c328491 100644 --- a/python/private/pypi/render_pkg_aliases.bzl +++ b/python/private/pypi/render_pkg_aliases.bzl @@ -22,8 +22,6 @@ load( ":generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel", ) # buildifier: disable=bzl-visibility -load(":parse_whl_name.bzl", "parse_whl_name") -load(":whl_target_platforms.bzl", "whl_target_platforms") NO_MATCH_ERROR_MESSAGE_TEMPLATE = """\ No matching wheel for current configuration's Python version. @@ -48,19 +46,17 @@ def _repr_dict(*, value_repr = repr, **kwargs): return {k: value_repr(v) for k, v in kwargs.items() if v} def _repr_config_setting(alias): - if alias.filename or alias.target_platforms: + if alias.target_platforms: return render.call( "whl_config_setting", **_repr_dict( - filename = alias.filename, target_platforms = alias.target_platforms, - config_setting = alias.config_setting, version = alias.version, ) ) else: return repr( - alias.config_setting or "//_config:is_cp{}".format(alias.version.replace(".", "")), + "//_config:is_cp{}".format(alias.version.replace(".", "")), ) def _repr_actual(aliases): @@ -179,15 +175,9 @@ def render_multiplatform_pkg_aliases(*, aliases, platform_config_settings = {}, contents = render_pkg_aliases( aliases = aliases, - glibc_versions = flag_versions.get("glibc_versions", []), - muslc_versions = flag_versions.get("muslc_versions", []), - osx_versions = flag_versions.get("osx_versions", []), **kwargs ) contents["_config/BUILD.bazel"] = _render_config_settings( - glibc_versions = flag_versions.get("glibc_versions", []), - muslc_versions = flag_versions.get("muslc_versions", []), - osx_versions = flag_versions.get("osx_versions", []), python_versions = _major_minor_versions(flag_versions.get("python_versions", [])), platform_config_settings = platform_config_settings, visibility = ["//:__subpackages__"], @@ -219,54 +209,21 @@ def get_whl_flag_versions(settings): * python_versions """ python_versions = {} - glibc_versions = {} target_platforms = {} - muslc_versions = {} - osx_versions = {} for setting in settings: - if not setting.version and not setting.filename: + if not setting.version: continue if setting.version: python_versions[setting.version] = None - if setting.filename and setting.filename.endswith(".whl") and not setting.filename.endswith("-any.whl"): - parsed = parse_whl_name(setting.filename) - else: - for plat in setting.target_platforms or []: - target_platforms[_non_versioned_platform(plat)] = None - continue - - for platform_tag in parsed.platform_tag.split("."): - parsed = whl_target_platforms(platform_tag) - - for p in parsed: - target_platforms[p.target_platform] = None - - if platform_tag.startswith("win") or platform_tag.startswith("linux"): - continue - - head, _, tail = platform_tag.partition("_") - major, _, tail = tail.partition("_") - minor, _, tail = tail.partition("_") - if tail: - version = (int(major), int(minor)) - if "many" in head: - glibc_versions[version] = None - elif "musl" in head: - muslc_versions[version] = None - elif "mac" in head: - osx_versions[version] = None - else: - fail(platform_tag) + for plat in setting.target_platforms or []: + target_platforms[_non_versioned_platform(plat)] = None return { k: sorted(v) for k, v in { - "glibc_versions": glibc_versions, - "muslc_versions": muslc_versions, - "osx_versions": osx_versions, "python_versions": python_versions, "target_platforms": target_platforms, }.items() diff --git a/python/private/pypi/whl_config_setting.bzl b/python/private/pypi/whl_config_setting.bzl index 3b81e4694f..1d868b1b65 100644 --- a/python/private/pypi/whl_config_setting.bzl +++ b/python/private/pypi/whl_config_setting.bzl @@ -14,7 +14,7 @@ "A small function to create an alias for a whl distribution" -def whl_config_setting(*, version = None, config_setting = None, filename = None, target_platforms = None): +def whl_config_setting(*, version = None, target_platforms = None): """The bzl_packages value used by by the render_pkg_aliases function. This contains the minimum amount of information required to generate correct @@ -25,15 +25,17 @@ def whl_config_setting(*, version = None, config_setting = None, filename = None whl alias is for. If not set, then non-version aware aliases will be constructed. This is mainly used for better error messages when there is no match found during a select. - config_setting: {type}`str | Label | None` the config setting that we should use. Defaults - to "//_config:is_python_{version}". - filename: {type}`str | None` the distribution filename to derive the config_setting. target_platforms: {type}`list[str] | None` the list of target_platforms for this distribution. Returns: a struct with the validated and parsed values. """ + + # FIXME @aignas 2025-07-26: There is still a potential that there will be ambigous match + # if the user is trying to have different packages for 3.X.Y and 3.X.Z versions of python + # in the same hub repository. Consider just removing this processing as I am not sure it + # has much value. if target_platforms: target_platforms_input = target_platforms target_platforms = [] @@ -50,8 +52,6 @@ def whl_config_setting(*, version = None, config_setting = None, filename = None target_platforms.append("{}_{}".format(abi, tail)) return struct( - config_setting = config_setting, - filename = filename, # Make the struct hashable target_platforms = tuple(target_platforms) if target_platforms else None, version = version, diff --git a/tests/pypi/config_settings/config_settings_tests.bzl b/tests/pypi/config_settings/config_settings_tests.bzl index a15f6b4d32..b3e6ada9e8 100644 --- a/tests/pypi/config_settings/config_settings_tests.bzl +++ b/tests/pypi/config_settings/config_settings_tests.bzl @@ -97,554 +97,6 @@ def _test_legacy_with_constraint_values(name): _tests.append(_test_legacy_with_constraint_values) -# Tests when we only have an `sdist` present. - -def _test_sdist_default(name): - _analysis_test( - name = name, - dist = { - "is_cp37_sdist": "sdist", - }, - want = "sdist", - ) - -_tests.append(_test_sdist_default) - -def _test_legacy_less_specialized_than_sdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37": "legacy", - "is_cp37_sdist": "sdist", - }, - want = "sdist", - ) - -_tests.append(_test_legacy_less_specialized_than_sdist) - -def _test_sdist_no_whl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("no"), - ], - want = "sdist", - ) - -_tests.append(_test_sdist_no_whl) - -def _test_sdist_no_sdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("only"), - ], - # We will use `no_match_error` in the real case to indicate that `sdist` is not - # allowed to be used. - want = "no_match", - ) - -_tests.append(_test_sdist_no_sdist) - -def _test_basic_whl_default(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py_none_any": "whl", - "is_cp37_sdist": "sdist", - }, - want = "whl", - ) - -_tests.append(_test_basic_whl_default) - -def _test_basic_whl_nowhl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py_none_any": "whl", - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("no"), - ], - want = "sdist", - ) - -_tests.append(_test_basic_whl_nowhl) - -def _test_basic_whl_nosdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py_none_any": "whl", - "is_cp37_sdist": "sdist", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("only"), - ], - want = "whl", - ) - -_tests.append(_test_basic_whl_nosdist) - -def _test_whl_default(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "whl", - "is_cp37_py_none_any": "basic_whl", - }, - want = "whl", - ) - -_tests.append(_test_whl_default) - -def _test_whl_nowhl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "whl", - "is_cp37_py_none_any": "basic_whl", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("no"), - ], - want = "no_match", - ) - -_tests.append(_test_whl_nowhl) - -def _test_whl_nosdist(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "whl", - }, - config_settings = [ - _flag.platform("linux_aarch64"), - _flag.pip_whl("only"), - ], - want = "whl", - ) - -_tests.append(_test_whl_nosdist) - -def _test_abi_whl_is_prefered(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_abi3_any": "abi_whl", - "is_cp37_py3_none_any": "whl", - }, - want = "abi_whl", - ) - -_tests.append(_test_abi_whl_is_prefered) - -def _test_whl_with_constraints_is_prefered(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_none_any": "default_whl", - "is_cp37_py3_none_any_linux_aarch64": "whl", - "is_cp37_py3_none_any_linux_x86_64": "amd64_whl", - }, - want = "whl", - ) - -_tests.append(_test_whl_with_constraints_is_prefered) - -def _test_cp_whl_is_prefered_over_py3(name): - _analysis_test( - name = name, - dist = { - "is_cp37_none_any": "cp", - "is_cp37_py3_abi3_any": "py3_abi3", - "is_cp37_py3_none_any": "py3", - }, - want = "cp", - ) - -_tests.append(_test_cp_whl_is_prefered_over_py3) - -def _test_cp_abi_whl_is_prefered_over_py3(name): - _analysis_test( - name = name, - dist = { - "is_cp37_abi3_any": "cp", - "is_cp37_py3_abi3_any": "py3", - }, - want = "cp", - ) - -_tests.append(_test_cp_abi_whl_is_prefered_over_py3) - -def _test_cp_version_is_selected_when_python_version_is_specified(name): - _analysis_test( - name = name, - dist = { - "is_cp310_none_any": "cp310", - "is_cp38_none_any": "cp38", - "is_cp39_none_any": "cp39", - }, - want = "cp310", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_cp_version_is_selected_when_python_version_is_specified) - -def _test_py_none_any_versioned(name): - _analysis_test( - name = name, - dist = { - "is_cp310_py_none_any": "whl", - "is_cp39_py_none_any": "too-low", - }, - want = "whl", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_py_none_any_versioned) - -def _test_cp_whl_is_not_prefered_over_py3_non_freethreaded(name): - _analysis_test( - name = name, - dist = { - "is_cp37_abi3_any": "py3_abi3", - "is_cp37_cp37t_any": "cp", - "is_cp37_none_any": "py3", - }, - want = "py3_abi3", - config_settings = [ - _flag.py_freethreaded("no"), - ], - ) - -_tests.append(_test_cp_whl_is_not_prefered_over_py3_non_freethreaded) - -def _test_cp_whl_is_not_prefered_over_py3_freethreaded(name): - _analysis_test( - name = name, - dist = { - "is_cp37_abi3_any": "py3_abi3", - "is_cp37_cp37_any": "cp", - "is_cp37_none_any": "py3", - }, - want = "py3", - config_settings = [ - _flag.py_freethreaded("yes"), - ], - ) - -_tests.append(_test_cp_whl_is_not_prefered_over_py3_freethreaded) - -def _test_cp_cp_whl(name): - _analysis_test( - name = name, - dist = { - "is_cp310_cp310_linux_aarch64": "whl", - }, - want = "whl", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_cp_cp_whl) - -def _test_cp_version_sdist_is_selected(name): - _analysis_test( - name = name, - dist = { - "is_cp310_sdist": "sdist", - }, - want = "sdist", - config_settings = [ - _flag.python_version("3.10.9"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_cp_version_sdist_is_selected) - -# NOTE: Right now there is no way to get the following behaviour without -# breaking other tests. We need to choose either ta have the correct -# specialization behaviour between `is_cp37_cp37_any` and -# `is_cp37_cp37_any_linux_aarch64` or this commented out test case. -# -# I think having this behaviour not working is fine because the `suffix` -# will be either present on all of config settings of the same platform -# or none, because we use it as a way to select a separate version of the -# wheel for a single platform only. -# -# If we can think of a better way to handle it, then we can lift this -# limitation. -# -# def _test_any_whl_with_suffix_specialization(name): -# _analysis_test( -# name = name, -# dist = { -# "is_cp37_abi3_any_linux_aarch64": "abi3", -# "is_cp37_cp37_any": "cp37", -# }, -# want = "cp37", -# ) -# -# _tests.append(_test_any_whl_with_suffix_specialization) - -def _test_platform_vs_any_with_suffix_specialization(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_any_linux_aarch64": "any", - "is_cp37_py3_none_linux_aarch64": "platform_whl", - }, - want = "platform_whl", - ) - -_tests.append(_test_platform_vs_any_with_suffix_specialization) - -def _test_platform_whl_is_prefered_over_any_whl_with_constraints(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_abi3_any": "better_default_whl", - "is_cp37_py3_abi3_any_linux_aarch64": "better_default_any_whl", - "is_cp37_py3_none_any": "default_whl", - "is_cp37_py3_none_any_linux_aarch64": "whl", - "is_cp37_py3_none_linux_aarch64": "platform_whl", - }, - want = "platform_whl", - ) - -_tests.append(_test_platform_whl_is_prefered_over_any_whl_with_constraints) - -def _test_abi3_platform_whl_preference(name): - _analysis_test( - name = name, - dist = { - "is_cp37_py3_abi3_linux_aarch64": "abi3_platform", - "is_cp37_py3_none_linux_aarch64": "platform", - }, - want = "abi3_platform", - ) - -_tests.append(_test_abi3_platform_whl_preference) - -def _test_glibc(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_manylinux_aarch64": "glibc", - "is_cp37_py3_abi3_linux_aarch64": "abi3_platform", - }, - want = "glibc", - ) - -_tests.append(_test_glibc) - -def _test_glibc_versioned(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_manylinux_2_14_aarch64": "glibc", - "is_cp37_cp37_manylinux_2_17_aarch64": "glibc", - "is_cp37_py3_abi3_linux_aarch64": "abi3_platform", - }, - want = "glibc", - config_settings = [ - _flag.py_linux_libc("glibc"), - _flag.pip_whl_glibc_version("2.17"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_glibc_versioned) - -def _test_glibc_compatible_exists(name): - _analysis_test( - name = name, - dist = { - # Code using the conditions will need to construct selects, which - # do the version matching correctly. - "is_cp37_cp37_manylinux_2_14_aarch64": "2_14_whl_via_2_14_branch", - "is_cp37_cp37_manylinux_2_17_aarch64": "2_14_whl_via_2_17_branch", - }, - want = "2_14_whl_via_2_17_branch", - config_settings = [ - _flag.py_linux_libc("glibc"), - _flag.pip_whl_glibc_version("2.17"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_glibc_compatible_exists) - -def _test_musl(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_musllinux_aarch64": "musl", - }, - want = "musl", - config_settings = [ - _flag.py_linux_libc("musl"), - _flag.platform("linux_aarch64"), - ], - ) - -_tests.append(_test_musl) - -def _test_windows(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_windows_x86_64": "whl", - "is_cp37_cp37t_windows_x86_64": "whl_freethreaded", - }, - want = "whl", - config_settings = [ - _flag.platform("windows_x86_64"), - ], - ) - -_tests.append(_test_windows) - -def _test_windows_freethreaded(name): - _analysis_test( - name = name, - dist = { - "is_cp37_cp37_windows_x86_64": "whl", - "is_cp37_cp37t_windows_x86_64": "whl_freethreaded", - }, - want = "whl_freethreaded", - config_settings = [ - _flag.platform("windows_x86_64"), - _flag.py_freethreaded("yes"), - ], - ) - -_tests.append(_test_windows_freethreaded) - -def _test_osx(name): - _analysis_test( - name = name, - dist = { - # We prefer arch specific whls over universal - "is_cp37_cp37_osx_universal2": "universal_whl", - "is_cp37_cp37_osx_x86_64": "whl", - }, - want = "whl", - config_settings = [ - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx) - -def _test_osx_universal_default(name): - _analysis_test( - name = name, - dist = { - # We default to universal if only that exists - "is_cp37_cp37_osx_universal2": "whl", - }, - want = "whl", - config_settings = [ - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx_universal_default) - -def _test_osx_universal_only(name): - _analysis_test( - name = name, - dist = { - # If we prefer universal, then we use that - "is_cp37_cp37_osx_universal2": "universal", - "is_cp37_cp37_osx_x86_64": "whl", - }, - want = "universal", - config_settings = [ - _flag.pip_whl_osx_arch("universal"), - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx_universal_only) - -def _test_osx_os_version(name): - _analysis_test( - name = name, - dist = { - # Similarly to the libc version, the user of the config settings will have to - # construct the select so that the version selection is correct. - "is_cp37_cp37_osx_10_9_x86_64": "whl", - }, - want = "whl", - config_settings = [ - _flag.pip_whl_osx_version("10.9"), - _flag.platform("mac_x86_64"), - ], - ) - -_tests.append(_test_osx_os_version) - -def _test_all(name): - _analysis_test( - name = name, - dist = { - "is_cp37_" + f: f - for f in [ - "{py}{abi}_{plat}".format(py = valid_py, abi = valid_abi, plat = valid_plat) - # we have py2.py3, py3, cp3 - for valid_py in ["py_", "py3_", ""] - # cp abi usually comes with a version and we only need one - # config setting variant for all of them because the python - # version will discriminate between different versions. - for valid_abi in ["none", "abi3", "cp37"] - for valid_plat in [ - "any", - "manylinux_2_17_x86_64", - "manylinux_2_17_aarch64", - "osx_x86_64", - "windows_x86_64", - ] - if not ( - valid_abi == "abi3" and valid_py == "py_" or - valid_abi == "cp37" and valid_py != "" - ) - ] - }, - want = "cp37_manylinux_2_17_x86_64", - config_settings = [ - _flag.pip_whl_glibc_version("2.17"), - _flag.platform("linux_x86_64"), - ], - ) - -_tests.append(_test_all) - def config_settings_test_suite(name): # buildifier: disable=function-docstring test_suite( name = name, @@ -654,9 +106,6 @@ def config_settings_test_suite(name): # buildifier: disable=function-docstring config_settings( name = "dummy", python_versions = ["3.7", "3.8", "3.9", "3.10"], - glibc_versions = [(2, 14), (2, 17)], - muslc_versions = [(1, 1)], - osx_versions = [(10, 9), (11, 0)], platform_config_settings = { "linux_aarch64": [ "@platforms//cpu:aarch64", diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 72cbb61d81..b6530a4df5 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -588,25 +588,25 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "torch": { "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [ whl_config_setting( - target_platforms = ["cp312_linux_x86_64"], + target_platforms = ("cp312_linux_x86_64",), version = "3.12", ), ], "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [ whl_config_setting( - target_platforms = ["cp312_linux_aarch64"], + target_platforms = ("cp312_linux_aarch64",), version = "3.12", ), ], "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [ whl_config_setting( - target_platforms = ["cp312_windows_x86_64"], + target_platforms = ("cp312_windows_x86_64",), version = "3.12", ), ], "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [ whl_config_setting( - target_platforms = ["cp312_osx_aarch64"], + target_platforms = ("cp312_osx_aarch64",), version = "3.12", ), ], @@ -1079,8 +1079,6 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' "cp315_linux_aarch64", "cp315_linux_x86_64", ], - config_setting = None, - filename = None, ), ], "pypi_315_optimum_osx_aarch64": [ @@ -1089,8 +1087,6 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' target_platforms = [ "cp315_osx_aarch64", ], - config_setting = None, - filename = None, ), ], }, @@ -1121,7 +1117,7 @@ def _test_pipstar_platforms(env): name = "rules_python", default = [ _default( - platform = "my{}_{}".format(os, cpu), + platform = "my{}{}".format(os, cpu), os_name = os, arch_name = cpu, config_settings = [ @@ -1161,19 +1157,19 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' pypi.hub_whl_map().contains_exactly({ "pypi": { "optimum": { - "pypi_315_optimum_mylinux_x86_64": [ + "pypi_315_optimum_mylinuxx86_64": [ whl_config_setting( version = "3.15", target_platforms = [ - "cp315_mylinux_x86_64", + "cp315_mylinuxx86_64", ], ), ], - "pypi_315_optimum_myosx_aarch64": [ + "pypi_315_optimum_myosxaarch64": [ whl_config_setting( version = "3.15", target_platforms = [ - "cp315_myosx_aarch64", + "cp315_myosxaarch64", ], ), ], @@ -1182,12 +1178,12 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' }) pypi.whl_libraries().contains_exactly({ - "pypi_315_optimum_mylinux_x86_64": { + "pypi_315_optimum_mylinuxx86_64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime-gpu]==1.17.1", }, - "pypi_315_optimum_myosx_aarch64": { + "pypi_315_optimum_myosxaarch64": { "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", "requirement": "optimum[onnxruntime]==1.17.1", diff --git a/tests/pypi/integration/BUILD.bazel b/tests/pypi/integration/BUILD.bazel index 9ea8dcebe4..316abe8271 100644 --- a/tests/pypi/integration/BUILD.bazel +++ b/tests/pypi/integration/BUILD.bazel @@ -1,20 +1,7 @@ load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_python_publish_deps//:requirements.bzl", "all_requirements") -load(":transitions.bzl", "transition_rule") build_test( name = "all_requirements_build_test", targets = all_requirements, ) - -# Rule that transitions dependencies to be built from sdist -transition_rule( - name = "all_requirements_from_sdist", - testonly = True, - deps = all_requirements, -) - -build_test( - name = "all_requirements_from_sdist_build_test", - targets = ["all_requirements_from_sdist"], -) diff --git a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl index 3fd08c393c..6248261ed4 100644 --- a/tests/pypi/pkg_aliases/pkg_aliases_test.bzl +++ b/tests/pypi/pkg_aliases/pkg_aliases_test.bzl @@ -159,18 +159,12 @@ def _test_multiplatform_whl_aliases(env): name = "bar_baz", actual = { whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", version = "3.9", - ): "filename_repo", + ): "version_repo", whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", version = "3.9", target_platforms = ["cp39_linux_x86_64"], - ): "filename_repo_for_platform", - whl_config_setting( - version = "3.9", - target_platforms = ["cp39_linux_x86_64"], - ): "bzlmod_repo_for_a_particular_platform", + ): "version_platform_repo", "//:my_config_setting": "bzlmod_repo", }, extra_aliases = [], @@ -178,18 +172,14 @@ def _test_multiplatform_whl_aliases(env): alias = lambda *, name, actual, visibility = None, tags = None: got.update({name: actual}), ), select = mock_select, - glibc_versions = [], - muslc_versions = [], - osx_versions = [], ) # buildifier: disable=unsorted-dict-items want = { "pkg": { "//:my_config_setting": "@bzlmod_repo//:pkg", - "//_config:is_cp39_linux_x86_64": "@bzlmod_repo_for_a_particular_platform//:pkg", - "//_config:is_cp39_py3_none_any": "@filename_repo//:pkg", - "//_config:is_cp39_py3_none_any_linux_x86_64": "@filename_repo_for_platform//:pkg", + "//_config:is_cp39": "@version_repo//:pkg", + "//_config:is_cp39_linux_x86_64": "@version_platform_repo//:pkg", "//conditions:default": "_no_matching_repository", }, } @@ -198,9 +188,8 @@ def _test_multiplatform_whl_aliases(env): env.expect.that_str(actual_no_match_error[0]).contains("""\ configuration settings: //:my_config_setting + //_config:is_cp39 //_config:is_cp39_linux_x86_64 - //_config:is_cp39_py3_none_any - //_config:is_cp39_py3_none_any_linux_x86_64 """) @@ -279,7 +268,6 @@ _tests.append(_test_multiplatform_whl_aliases_nofilename) def _test_multiplatform_whl_aliases_nofilename_target_platforms(env): aliases = { whl_config_setting( - config_setting = "//:ignored", version = "3.1", target_platforms = [ "cp31_linux_x86_64", @@ -298,102 +286,6 @@ def _test_multiplatform_whl_aliases_nofilename_target_platforms(env): _tests.append(_test_multiplatform_whl_aliases_nofilename_target_platforms) -def _test_multiplatform_whl_aliases_filename(env): - aliases = { - whl_config_setting( - filename = "foo-0.0.3-py3-none-any.whl", - version = "3.2", - ): "foo-py3-0.0.3", - whl_config_setting( - filename = "foo-0.0.1-py3-none-any.whl", - version = "3.1", - ): "foo-py3-0.0.1", - whl_config_setting( - filename = "foo-0.0.1-cp313-cp313-any.whl", - version = "3.13", - ): "foo-cp-0.0.1", - whl_config_setting( - filename = "foo-0.0.1-cp313-cp313t-any.whl", - version = "3.13", - ): "foo-cpt-0.0.1", - whl_config_setting( - filename = "foo-0.0.2-py3-none-any.whl", - version = "3.1", - target_platforms = [ - "cp31_linux_x86_64", - "cp31_linux_aarch64", - ], - ): "foo-0.0.2", - } - got = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = [], - muslc_versions = [], - osx_versions = [], - ) - want = { - "//_config:is_cp313_cp313_any": "foo-cp-0.0.1", - "//_config:is_cp313_cp313t_any": "foo-cpt-0.0.1", - "//_config:is_cp31_py3_none_any": "foo-py3-0.0.1", - "//_config:is_cp31_py3_none_any_linux_aarch64": "foo-0.0.2", - "//_config:is_cp31_py3_none_any_linux_x86_64": "foo-0.0.2", - "//_config:is_cp32_py3_none_any": "foo-py3-0.0.3", - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_multiplatform_whl_aliases_filename) - -def _test_multiplatform_whl_aliases_filename_versioned(env): - aliases = { - whl_config_setting( - filename = "foo-0.0.1-py3-none-manylinux_2_17_x86_64.whl", - version = "3.1", - ): "glibc-2.17", - whl_config_setting( - filename = "foo-0.0.1-py3-none-manylinux_2_18_x86_64.whl", - version = "3.1", - ): "glibc-2.18", - whl_config_setting( - filename = "foo-0.0.1-py3-none-musllinux_1_1_x86_64.whl", - version = "3.1", - ): "musl-1.1", - } - got = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = [(2, 17), (2, 18)], - muslc_versions = [(1, 1), (1, 2)], - osx_versions = [], - ) - want = { - # This could just work with: - # select({ - # "//_config:is_gt_eq_2.18": "//_config:is_cp3.1_py3_none_manylinux_x86_64", - # "//conditions:default": "//_config:is_gt_eq_2.18", - # }): "glibc-2.18", - # select({ - # "//_config:is_range_2.17_2.18": "//_config:is_cp3.1_py3_none_manylinux_x86_64", - # "//_config:is_glibc_default": "//_config:is_cp3.1_py3_none_manylinux_x86_64", - # "//conditions:default": "//_config:is_glibc_default", - # }): "glibc-2.17", - # ( - # "//_config:is_gt_musl_1.1": "musl-1.1", - # "//_config:is_musl_default": "musl-1.1", - # ): "musl-1.1", - # - # For this to fully work we need to have the pypi:config_settings.bzl to generate the - # extra targets that use the FeatureFlagInfo and this to generate extra aliases for the - # config settings. - "//_config:is_cp31_py3_none_manylinux_2_17_x86_64": "glibc-2.17", - "//_config:is_cp31_py3_none_manylinux_2_18_x86_64": "glibc-2.18", - "//_config:is_cp31_py3_none_manylinux_x86_64": "glibc-2.17", - "//_config:is_cp31_py3_none_musllinux_1_1_x86_64": "musl-1.1", - "//_config:is_cp31_py3_none_musllinux_1_2_x86_64": "musl-1.1", - "//_config:is_cp31_py3_none_musllinux_x86_64": "musl-1.1", - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_multiplatform_whl_aliases_filename_versioned) - def _mock_alias(container): return lambda name, **kwargs: container.append(name) @@ -451,86 +343,6 @@ def _test_config_settings_exist_legacy(env): _tests.append(_test_config_settings_exist_legacy) -def _test_config_settings_exist(env): - for py_tag in ["py2.py3", "py3", "py311", "cp311"]: - if py_tag == "py2.py3": - abis = ["none"] - elif py_tag.startswith("py"): - abis = ["none", "abi3"] - else: - abis = ["none", "abi3", "cp311"] - - for abi_tag in abis: - for platform_tag, kwargs in { - "any": {}, - "macosx_11_0_arm64": { - "osx_versions": [(11, 0)], - "platform_config_settings": { - "osx_aarch64": [ - "@platforms//cpu:aarch64", - "@platforms//os:osx", - ], - }, - }, - "manylinux_2_17_x86_64": { - "glibc_versions": [(2, 17), (2, 18)], - "platform_config_settings": { - "linux_x86_64": [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - }, - }, - "manylinux_2_18_x86_64": { - "glibc_versions": [(2, 17), (2, 18)], - "platform_config_settings": { - "linux_x86_64": [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - }, - }, - "musllinux_1_1_aarch64": { - "muslc_versions": [(1, 2), (1, 1), (1, 0)], - "platform_config_settings": { - "linux_aarch64": [ - "@platforms//cpu:aarch64", - "@platforms//os:linux", - ], - }, - }, - }.items(): - aliases = { - whl_config_setting( - filename = "foo-0.0.1-{}-{}-{}.whl".format(py_tag, abi_tag, platform_tag), - version = "3.11", - ): "repo", - } - available_config_settings = [] - config_settings( - python_versions = ["3.11"], - native = struct( - alias = _mock_alias(available_config_settings), - config_setting = _mock_config_setting([]), - ), - selects = struct( - config_setting_group = _mock_config_setting_group(available_config_settings), - ), - **kwargs - ) - - got_aliases = multiplatform_whl_aliases( - aliases = aliases, - glibc_versions = kwargs.get("glibc_versions", []), - muslc_versions = kwargs.get("muslc_versions", []), - osx_versions = kwargs.get("osx_versions", []), - ) - got = [a.partition(":")[-1] for a in got_aliases] - - env.expect.that_collection(available_config_settings).contains_at_least(got) - -_tests.append(_test_config_settings_exist) - def pkg_aliases_test_suite(name): """Create the test suite. diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl index ad7f36aed6..9114f59279 100644 --- a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl +++ b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl @@ -15,10 +15,6 @@ """render_pkg_aliases tests""" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load( - "//python/private/pypi:pkg_aliases.bzl", - "get_filename_config_settings", -) # buildifier: disable=bzl-visibility load( "//python/private/pypi:render_pkg_aliases.bzl", "get_whl_flag_versions", @@ -70,26 +66,13 @@ def _test_bzlmod_aliases(env): whl_config_setting( # Add one with micro version to mimic construction in the extension version = "3.2.2", - config_setting = "//:my_config_setting", ): "pypi_32_bar_baz", whl_config_setting( version = "3.2", - config_setting = "//:my_config_setting", target_platforms = [ "cp32_linux_x86_64", ], ): "pypi_32_bar_baz_linux_x86_64", - whl_config_setting( - version = "3.2", - filename = "foo-0.0.0-py3-none-any.whl", - ): "filename_repo", - whl_config_setting( - version = "3.2.2", - filename = "foo-0.0.0-py3-none-any.whl", - target_platforms = [ - "cp32.2_linux_x86_64", - ], - ): "filename_repo_linux_x86_64", }, }, extra_hub_aliases = {"bar_baz": ["foo"]}, @@ -111,21 +94,11 @@ package(default_visibility = ["//visibility:public"]) pkg_aliases( name = "bar_baz", actual = { - "//:my_config_setting": "pypi_32_bar_baz", + "//_config:is_cp322": "pypi_32_bar_baz", whl_config_setting( target_platforms = ("cp32_linux_x86_64",), - config_setting = "//:my_config_setting", version = "3.2", ): "pypi_32_bar_baz_linux_x86_64", - whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", - version = "3.2", - ): "filename_repo", - whl_config_setting( - filename = "foo-0.0.0-py3-none-any.whl", - target_platforms = ("cp32_linux_x86_64",), - version = "3.2.2", - ): "filename_repo_linux_x86_64", }, extra_aliases = ["foo"], )""" @@ -256,252 +229,24 @@ def _test_get_python_versions_with_target_platforms(env): _tests.append(_test_get_python_versions_with_target_platforms) -def _test_get_python_versions_from_filenames(env): - got = get_whl_flag_versions( - settings = [ - whl_config_setting( - version = "3.3", - filename = "foo-0.0.0-py3-none-" + plat + ".whl", - ) - for plat in [ - "linux_x86_64", - "manylinux_2_17_x86_64", - "manylinux_2_14_aarch64.musllinux_1_1_aarch64", - "musllinux_1_0_x86_64", - "manylinux2014_x86_64.manylinux_2_17_x86_64", - "macosx_11_0_arm64", - "macosx_10_9_x86_64", - "macosx_10_9_universal2", - "windows_x86_64", - ] - ], - ) - want = { - "glibc_versions": [(2, 14), (2, 17)], - "muslc_versions": [(1, 0), (1, 1)], - "osx_versions": [(10, 9), (11, 0)], - "python_versions": ["3.3"], - "target_platforms": [ - "linux_aarch64", - "linux_x86_64", - "osx_aarch64", - "osx_x86_64", - "windows_x86_64", - ], - } - env.expect.that_dict(got).contains_exactly(want) - -_tests.append(_test_get_python_versions_from_filenames) - def _test_get_flag_versions_from_alias_target_platforms(env): got = get_whl_flag_versions( settings = [ + whl_config_setting(version = "3.3"), whl_config_setting( version = "3.3", - filename = "foo-0.0.0-py3-none-" + plat + ".whl", - ) - for plat in [ - "windows_x86_64", - ] - ] + [ - whl_config_setting( - version = "3.3", - filename = "foo-0.0.0-py3-none-any.whl", - target_platforms = [ - "cp33_linux_x86_64", - ], + target_platforms = ["cp33_linux_x86_64"], ), ], ) want = { "python_versions": ["3.3"], - "target_platforms": [ - "linux_x86_64", - "windows_x86_64", - ], + "target_platforms": ["linux_x86_64"], } env.expect.that_dict(got).contains_exactly(want) _tests.append(_test_get_flag_versions_from_alias_target_platforms) -def _test_config_settings( - env, - *, - filename, - want, - python_version, - want_versions = {}, - target_platforms = [], - glibc_versions = [], - muslc_versions = [], - osx_versions = []): - got, got_default_version_settings = get_filename_config_settings( - filename = filename, - target_platforms = target_platforms, - glibc_versions = glibc_versions, - muslc_versions = muslc_versions, - osx_versions = osx_versions, - python_version = python_version, - ) - env.expect.that_collection(got).contains_exactly(want) - env.expect.that_dict(got_default_version_settings).contains_exactly(want_versions) - -def _test_sdist(env): - # Do the first test for multiple extensions - for ext in [".tar.gz", ".zip"]: - _test_config_settings( - env, - filename = "foo-0.0.1" + ext, - python_version = "3.2", - want = [":is_cp32_sdist"], - ) - - ext = ".zip" - _test_config_settings( - env, - filename = "foo-0.0.1" + ext, - python_version = "3.2", - target_platforms = [ - "linux_aarch64", - "linux_x86_64", - ], - want = [ - ":is_cp32_sdist_linux_aarch64", - ":is_cp32_sdist_linux_x86_64", - ], - ) - -_tests.append(_test_sdist) - -def _test_py2_py3_none_any(env): - _test_config_settings( - env, - filename = "foo-0.0.1-py2.py3-none-any.whl", - python_version = "3.2", - want = [ - ":is_cp32_py_none_any", - ], - ) - - _test_config_settings( - env, - filename = "foo-0.0.1-py2.py3-none-any.whl", - python_version = "3.2", - target_platforms = [ - "osx_x86_64", - ], - want = [":is_cp32_py_none_any_osx_x86_64"], - ) - -_tests.append(_test_py2_py3_none_any) - -def _test_py3_none_any(env): - _test_config_settings( - env, - filename = "foo-0.0.1-py3-none-any.whl", - python_version = "3.1", - want = [":is_cp31_py3_none_any"], - ) - - _test_config_settings( - env, - filename = "foo-0.0.1-py3-none-any.whl", - python_version = "3.1", - target_platforms = ["linux_x86_64"], - want = [":is_cp31_py3_none_any_linux_x86_64"], - ) - -_tests.append(_test_py3_none_any) - -def _test_py3_none_macosx_10_9_universal2(env): - _test_config_settings( - env, - filename = "foo-0.0.1-py3-none-macosx_10_9_universal2.whl", - python_version = "3.1", - osx_versions = [ - (10, 9), - (11, 0), - ], - want = [], - want_versions = { - ":is_cp31_py3_none_osx_universal2": { - (10, 9): ":is_cp31_py3_none_osx_10_9_universal2", - (11, 0): ":is_cp31_py3_none_osx_11_0_universal2", - }, - }, - ) - -_tests.append(_test_py3_none_macosx_10_9_universal2) - -def _test_cp37_abi3_linux_x86_64(env): - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-abi3-linux_x86_64.whl", - python_version = "3.7", - want = [":is_cp37_abi3_linux_x86_64"], - ) - -_tests.append(_test_cp37_abi3_linux_x86_64) - -def _test_cp37_abi3_windows_x86_64(env): - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-abi3-windows_x86_64.whl", - python_version = "3.7", - want = [":is_cp37_abi3_windows_x86_64"], - ) - -_tests.append(_test_cp37_abi3_windows_x86_64) - -def _test_cp37_abi3_manylinux_2_17_x86_64(env): - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", - python_version = "3.7", - glibc_versions = [ - (2, 16), - (2, 17), - (2, 18), - ], - want = [], - want_versions = { - ":is_cp37_abi3_manylinux_x86_64": { - (2, 17): ":is_cp37_abi3_manylinux_2_17_x86_64", - (2, 18): ":is_cp37_abi3_manylinux_2_18_x86_64", - }, - }, - ) - -_tests.append(_test_cp37_abi3_manylinux_2_17_x86_64) - -def _test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64(env): - # I've seen such a wheel being built for `uv` - _test_config_settings( - env, - filename = "foo-0.0.1-cp37-cp37-manylinux_2_17_arm64.musllinux_1_1_arm64.whl", - python_version = "3.7", - glibc_versions = [ - (2, 16), - (2, 17), - (2, 18), - ], - muslc_versions = [ - (1, 1), - ], - want = [], - want_versions = { - ":is_cp37_cp37_manylinux_aarch64": { - (2, 17): ":is_cp37_cp37_manylinux_2_17_aarch64", - (2, 18): ":is_cp37_cp37_manylinux_2_18_aarch64", - }, - ":is_cp37_cp37_musllinux_aarch64": { - (1, 1): ":is_cp37_cp37_musllinux_1_1_aarch64", - }, - }, - ) - -_tests.append(_test_cp37_abi3_manylinux_2_17_musllinux_1_1_aarch64) - def render_pkg_aliases_test_suite(name): """Create the test suite.