feat(bzlmod): support wheel-only additions to the hub repo

.bazelrc

@@ -4,8 +4,8 @@
 # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it)
 # To update these lines, execute
 # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages`
-build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
-query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
+query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/dupe_requirements,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,gazelle,gazelle/manifest,gazelle/manifest/generate,gazelle/manifest/hasher,gazelle/manifest/test,gazelle/modules_mapping,gazelle/python,gazelle/pythonconfig,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/pip_repository_entry_points,tests/integration/py_cc_toolchain_registered
 
 test --test_output=errors
 

CHANGELOG.md

@@ -28,6 +28,11 @@ A brief description of the categories of changes:
 * New Python versions available: `3.11.8`, `3.12.2` using
   https://github.com/indygreg/python-build-standalone/releases/tag/20240224.
 
+* (bzlmod pip) One can add hub repositories that are for a specific pip platform
+  by specifying the `experimental_platform` atribute on the `pip.parse` tag class.
+  This effectively turn on the `download_only = True` flag and setup the extra
+  `pip` args based on the `python_version` value.
+
 [0.XX.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.XX.0
 
 ## [0.31.0] - 2024-02-12
@@ -115,7 +120,6 @@ A brief description of the categories of changes:
 * (coverage) Add support for python 3.12 and bump `coverage.py` to
   7.4.1.
 
-
 ## [0.29.0] - 2024-01-22
 
 [0.29.0]: https://github.com/bazelbuild/rules_python/releases/tag/0.29.0

examples/bzlmod/BUILD.bazel

@@ -92,3 +92,13 @@ build_test(
     name = "all_requirements",
     targets = all_requirements,
 )
+
+# Use the following to fetch wheels for a particular platform
+# bazel build @pip//... --platforms=//:linux_x86_6
+platform(
+    name = "linux_x86_64",
+    constraint_values = [
+        "@platforms//os:linux",
+        "@platforms//cpu:x86_64",
+    ],
+)

examples/bzlmod/MODULE.bazel

@@ -124,6 +124,38 @@ pip.parse(
         "@whl_mods_hub//:wheel.json": "wheel",
     },
 )
+
+# To setup fetching wheels for a particular target platform use the following:
+pip.parse(
+    experimental_requirement_cycles = {
+        "sphinx": [
+            "sphinx",
+            "sphinxcontrib-qthelp",
+            "sphinxcontrib-htmlhelp",
+            "sphinxcontrib-devhelp",
+            "sphinxcontrib-applehelp",
+            "sphinxcontrib-serializinghtml",
+        ],
+    },
+    # NOTE @aignas 2024-02-05: the experimental_target_platforms will be
+    # derived from the value below, so you do not need to setup anything.
+    experimental_whl_platform = "manylinux_2_17_x86_64",
+    # You can also set a particular config setting if the default one will not suffice
+    # or if you have a custom interpreter that is passed to pip.parse and you for example
+    # want to target a `musl` python distribution.
+    #experimental_platform_config_setting = "//:my_python_3.9_linux_x86_64",
+    hub_name = "pip",
+    python_version = "3.9",
+    # Specify the requirements file for the target platform
+    requirements_lock = "//:requirements_lock_3_9.txt",
+    # These modifications were created above and we
+    # are providing pip.parse with the label of the mod
+    # and the name of the wheel.
+    whl_modifications = {
+        "@whl_mods_hub//:requests.json": "requests",
+        "@whl_mods_hub//:wheel.json": "wheel",
+    },
+)
 pip.parse(
     experimental_requirement_cycles = {
         "sphinx": [

python/config_settings/BUILD.bazel

@@ -1,4 +1,3 @@
-load("//python:versions.bzl", "TOOL_VERSIONS")
 load(":config_settings.bzl", "construct_config_settings")
 
 filegroup(
@@ -10,7 +9,4 @@ filegroup(
     visibility = ["//python:__pkg__"],
 )
 
-construct_config_settings(
-    name = "construct_config_settings",
-    python_versions = TOOL_VERSIONS.keys(),
-)
+construct_config_settings()

python/config_settings/config_settings.bzl

@@ -15,105 +15,38 @@
 """This module is used to construct the config settings in the BUILD file in this same package.
 """
 
-load("@bazel_skylib//lib:selects.bzl", "selects")
 load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
-load("//python:versions.bzl", "MINOR_MAPPING")
+load("//python/private:config_settings.bzl", _VERSION_FLAG_VALUES = "VERSION_FLAG_VALUES", _is_python_config_setting = "is_python_config_setting")
 
-def construct_config_settings(name, python_versions):
-    """Constructs a set of configs for all Python versions.
-
-    Args:
-        name: str, unused; only specified to satisfy buildifier lint checks
-            and allow programatic modification of the target.
-        python_versions: list of all (x.y.z) Python versions supported by rules_python.
-    """
-
-    # Maps e.g. "3.8" -> ["3.8.1", "3.8.2", etc]
-    minor_to_micro_versions = {}
-
-    allowed_flag_values = []
-    for micro_version in python_versions:
-        minor, _, _ = micro_version.rpartition(".")
-        minor_to_micro_versions.setdefault(minor, []).append(micro_version)
-        allowed_flag_values.append(micro_version)
-
-    allowed_flag_values.extend(list(minor_to_micro_versions))
+VERSION_FLAG_VALUES = _VERSION_FLAG_VALUES
+is_python_config_setting = _is_python_config_setting
 
+def construct_config_settings(name = None):  # buildifier: disable=function-docstring
     string_flag(
         name = "python_version",
         # TODO: The default here should somehow match the MODULE config. Until
         # then, use the empty string to indicate an unknown version. This
         # also prevents version-unaware targets from inadvertently matching
         # a select condition when they shouldn't.
         build_setting_default = "",
-        values = [""] + sorted(allowed_flag_values),
+        values = [""] + VERSION_FLAG_VALUES.keys(),
         visibility = ["//visibility:public"],
     )
 
-    for minor_version, micro_versions in minor_to_micro_versions.items():
-        # This matches the raw flag value, e.g. --//python/config_settings:python_version=3.8
-        # It's private because matching the concept of e.g. "3.8" value is done
-        # using the `is_python_X.Y` config setting group, which is aware of the
-        # minor versions that could match instead.
-        equals_minor_version_name = "_python_version_flag_equals_" + minor_version
-        native.config_setting(
-            name = equals_minor_version_name,
-            flag_values = {":python_version": minor_version},
-        )
-        matches_minor_version_names = [equals_minor_version_name]
-
-        default_micro_version = MINOR_MAPPING[minor_version]
-
-        for micro_version in micro_versions:
-            is_micro_version_name = "is_python_" + micro_version
-            if default_micro_version != micro_version:
-                native.config_setting(
-                    name = is_micro_version_name,
-                    flag_values = {":python_version": micro_version},
-                    visibility = ["//visibility:public"],
-                )
-                matches_minor_version_names.append(is_micro_version_name)
-                continue
-
-            # Ensure that is_python_3.9.8 is matched if python_version is set
-            # to 3.9 if MINOR_MAPPING points to 3.9.8
-            equals_micro_name = "_python_version_flag_equals_" + micro_version
-            native.config_setting(
-                name = equals_micro_name,
-                flag_values = {":python_version": micro_version},
-            )
-
-            # An alias pointing to an underscore-prefixed config_setting_group
-            # is used because config_setting_group creates
-            # `is_{minor}_N` targets, which are easily confused with the
-            # `is_{minor}.{micro}` (dot) targets.
-            selects.config_setting_group(
-                name = "_" + is_micro_version_name,
-                match_any = [
-                    equals_micro_name,
-                    equals_minor_version_name,
-                ],
-            )
-            native.alias(
-                name = is_micro_version_name,
-                actual = "_" + is_micro_version_name,
-                visibility = ["//visibility:public"],
-            )
-            matches_minor_version_names.append(equals_micro_name)
-
-        # This is prefixed with an underscore to prevent confusion due to how
-        # config_setting_group is implemented and how our micro-version targets
-        # are named. config_setting_group will generate targets like
-        # "is_python_3.10_1" (where the `_N` suffix is len(match_any).
-        # Meanwhile, the micro-version tarets are named "is_python_3.10.1" --
-        # just a single dot vs underscore character difference.
-        selects.config_setting_group(
-            name = "_is_python_" + minor_version,
-            match_any = matches_minor_version_names,
-        )
-
-        native.alias(
-            name = "is_python_" + minor_version,
-            actual = "_is_python_" + minor_version,
+    for version, matching_versions in VERSION_FLAG_VALUES.items():
+        match_any = None
+        if len(matching_versions) > 1:
+            match_any = [
+                # Use the internal labels created by this macro in order to handle matching
+                # 3.8 config value if using the 3.8 version from MINOR_MAPPING with generating
+                # fewer targets overall.
+                ("_is_python_{}" if len(VERSION_FLAG_VALUES[v]) > 1 else "is_python_{}").format(v)
+                for v in matching_versions
+            ]
+
+        is_python_config_setting(
+            name = "is_python_{}".format(version),
+            python_version = version,
+            match_any = match_any,
             visibility = ["//visibility:public"],
         )

python/pip_install/pip_repository.bzl

@@ -798,9 +798,12 @@ def _whl_library_impl(rctx):
     target_platforms = rctx.attr.experimental_target_platforms
     if target_platforms:
         parsed_whl = parse_whl_name(whl_path.basename)
-        if parsed_whl.platform_tag != "any":
+        if parsed_whl.platform_tag != "any" and parsed_whl.abi_tag not in ["abi3", "none"]:
             # NOTE @aignas 2023-12-04: if the wheel is a platform specific
-            # wheel, we only include deps for that target platform
+            # wheel, we only include deps for that target platform, however
+            # we do this only if the abi is cp3Y, because otherwise we may to
+            # be able to use the same wheel and generate deps for multiple Python
+            # versions.
             target_platforms = [
                 "{}_{}_{}".format(parsed_whl.abi_tag, p.os, p.cpu)
                 for p in whl_target_platforms(parsed_whl.platform_tag)
@@ -855,6 +858,7 @@ def _whl_library_impl(rctx):
         ],
         entry_points = entry_points,
         annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
+        use_hub = rctx.attr.experimental_use_hub_for_deps,
     )
     rctx.file("BUILD.bazel", build_file_contents)
 
@@ -896,6 +900,9 @@ whl_library_attrs = {
         ),
         allow_files = True,
     ),
+    "experimental_use_hub_for_deps": attr.string(
+        default = "",
+    ),
     "group_deps": attr.string_list(
         doc = "List of dependencies to skip in order to break the cycles within a dependency group.",
         default = [],

python/pip_install/private/generate_whl_library_build_bazel.bzl

@@ -232,7 +232,8 @@ def generate_whl_library_build_bazel(
         entry_points,
         annotation = None,
         group_name = None,
-        group_deps = []):
+        group_deps = [],
+        use_hub = False):
     """Generate a BUILD file for an unzipped Wheel
 
     Args:
@@ -250,6 +251,7 @@ def generate_whl_library_build_bazel(
         group_deps: List[str]; names of fellow members of the group (if any). These will be excluded
           from generated deps lists so as to avoid direct cycles. These dependencies will be provided
           at runtime by the group rules which wrap this library and its fellows together.
+        use_hub: str; the hub to use instead of spoke repos when pointing to dependencies.
 
     Returns:
         A complete BUILD file as a string
@@ -338,13 +340,13 @@ def generate_whl_library_build_bazel(
     lib_dependencies = _render_list_and_select(
         deps = dependencies,
         deps_by_platform = dependencies_by_platform,
-        tmpl = "@{}{{}}//:{}".format(repo_prefix, PY_LIBRARY_PUBLIC_LABEL),
+        tmpl = "@{}//{{}}".format(use_hub) if use_hub else "@{}{{}}//:{}".format(repo_prefix, PY_LIBRARY_PUBLIC_LABEL),
     )
 
     whl_file_deps = _render_list_and_select(
         deps = dependencies,
         deps_by_platform = dependencies_by_platform,
-        tmpl = "@{}{{}}//:{}".format(repo_prefix, WHEEL_FILE_PUBLIC_LABEL),
+        tmpl = "@{}//{{}}:{}".format(use_hub, WHEEL_FILE_PUBLIC_LABEL) if use_hub else "@{}{{}}//:{}".format(repo_prefix, WHEEL_FILE_PUBLIC_LABEL),
     )
 
     # If this library is a member of a group, its public label aliases need to

python/private/BUILD.bazel

@@ -68,6 +68,15 @@ bzl_library(
     srcs = ["bzlmod_enabled.bzl"],
 )
 
+bzl_library(
+    name = "config_settings_bzl",
+    srcs = ["config_settings.bzl"],
+    deps = [
+        "//python:versions_bzl",
+        "@bazel_skylib//lib:selects",
+    ],
+)
+
 bzl_library(
     name = "coverage_deps_bzl",
     srcs = ["coverage_deps.bzl"],

python/private/bzlmod/BUILD.bazel

@@ -30,12 +30,14 @@ bzl_library(
     srcs = ["pip.bzl"],
     deps = [
         ":pip_repository_bzl",
+        ":utils_bzl",
         "//python/pip_install:pip_repository_bzl",
         "//python/pip_install:requirements_parser_bzl",
         "//python/private:full_version_bzl",
         "//python/private:normalize_name_bzl",
         "//python/private:parse_whl_name_bzl",
         "//python/private:version_label_bzl",
+        "//python/private:whl_target_platforms_bzl",
         ":bazel_features_bzl",
     ] + [
         "@pythons_hub//:interpreters_bzl",
@@ -52,6 +54,7 @@ bzl_library(
     srcs = ["pip_repository.bzl"],
     visibility = ["//:__subpackages__"],
     deps = [
+        ":utils_bzl",
         "//python/private:render_pkg_aliases_bzl",
         "//python/private:text_util_bzl",
     ],
@@ -76,3 +79,8 @@ bzl_library(
         "//python/private:py_toolchain_suite_bzl",
     ],
 )
+
+bzl_library(
+    name = "utils_bzl",
+    srcs = ["utils.bzl"],
+)