poc of accessing deps via the hub repo

Author: aignas

python/pip_install/pip_repository.bzl

@@ -858,6 +858,7 @@ def _whl_library_impl(rctx):
         ],
         entry_points = entry_points,
         annotation = None if not rctx.attr.annotation else struct(**json.decode(rctx.read(rctx.attr.annotation))),
+        use_hub = rctx.attr.experimental_use_hub_for_deps,
     )
     rctx.file("BUILD.bazel", build_file_contents)
 
@@ -899,6 +900,9 @@ whl_library_attrs = {
         ),
         allow_files = True,
     ),
+    "experimental_use_hub_for_deps": attr.string(
+        default = "",
+    ),
     "group_deps": attr.string_list(
         doc = "List of dependencies to skip in order to break the cycles within a dependency group.",
         default = [],

python/pip_install/private/generate_whl_library_build_bazel.bzl

@@ -232,7 +232,8 @@ def generate_whl_library_build_bazel(
         entry_points,
         annotation = None,
         group_name = None,
-        group_deps = []):
+        group_deps = [],
+        use_hub = False):
     """Generate a BUILD file for an unzipped Wheel
 
     Args:
@@ -250,6 +251,7 @@ def generate_whl_library_build_bazel(
         group_deps: List[str]; names of fellow members of the group (if any). These will be excluded
           from generated deps lists so as to avoid direct cycles. These dependencies will be provided
           at runtime by the group rules which wrap this library and its fellows together.
+        use_hub: str; the hub to use instead of spoke repos when pointing to dependencies.
 
     Returns:
         A complete BUILD file as a string
@@ -338,13 +340,13 @@ def generate_whl_library_build_bazel(
     lib_dependencies = _render_list_and_select(
         deps = dependencies,
         deps_by_platform = dependencies_by_platform,
-        tmpl = "@{}{{}}//:{}".format(repo_prefix, PY_LIBRARY_PUBLIC_LABEL),
+        tmpl = "@{}//{{}}".format(use_hub) if use_hub else "@{}{{}}//:{}".format(repo_prefix, PY_LIBRARY_PUBLIC_LABEL),
     )
 
     whl_file_deps = _render_list_and_select(
         deps = dependencies,
         deps_by_platform = dependencies_by_platform,
-        tmpl = "@{}{{}}//:{}".format(repo_prefix, WHEEL_FILE_PUBLIC_LABEL),
+        tmpl = "@{}//{{}}:{}".format(use_hub, WHEEL_FILE_PUBLIC_LABEL) if use_hub else "@{}{{}}//:{}".format(repo_prefix, WHEEL_FILE_PUBLIC_LABEL),
     )
 
     # If this library is a member of a group, its public label aliases need to

python/private/bzlmod/pip.bzl

@@ -121,33 +121,27 @@ def _create_whl_repos(*, module_ctx, pip_attr, whl_map, whl_overrides, config_se
             ))
         python_interpreter_target = INTERPRETER_LABELS[python_name]
 
-    if pip_attr.experimental_whl_platform:
-        pip_name = "{}_{}_{}".format(
-            hub_name,
-            pip_attr.experimental_whl_platform,
-            version_label(pip_attr.python_version),
-        )
-    else:
-        pip_name = "{}_{}".format(
-            hub_name,
-            version_label(pip_attr.python_version),
-        )
+    pip_name = "{}_{}".format(
+        hub_name,
+        version_label(pip_attr.python_version),
+    )
 
     major_minor = _major_minor_version(pip_attr.python_version)
     target_platforms = pip_attr.experimental_target_platforms
     platform_config_settings = []
+    default_version = _major_minor_version(DEFAULT_PYTHON_VERSION)
+
     if pip_attr.experimental_whl_platform:
         # NOTE @aignas 2024-02-16: Right now the download is extremely inefficient, because
         # we have to download wheels for the target platform from scratch even though some
         # of the wheels for the target platform could already be reused because they are
         # multi-platform.
         #
-        # In order to improve this we would have to make `whl_library` support accessing its
-        # deps via the hub_repo aliases (so that it references `@pip//foo` and does not need
-        # to know if `foo` is target_platform specific or not.  One more thing would be to
-        # somehow know in advance the filename of the `whl` for the `whl_library` that we are
-        # going to create. That way we could create only a single `whl_library` for all python
-        # versions/platforms at once, thus simplifying the hub repo layout.
+        # In order to improve this we would have to somehow know in advance the
+        # filename of the `whl` for the `whl_library` that we are going to
+        # create. That way we could create only a single `whl_library` for all
+        # python versions/platforms at once, thus simplifying the hub repo
+        # layout.
         requirements_lock = pip_attr.requirements_lock
         if not requirements_lock or pip_attr.requirements_windows or pip_attr.requirements_darwin or pip_attr.requirements_linux:
             fail("only requirements_lock can be specified when platform is used")
@@ -178,7 +172,6 @@ def _create_whl_repos(*, module_ctx, pip_attr, whl_map, whl_overrides, config_se
         # Handle default version matching as well, so that we could use the
         # platform specific wheels if they match the right platform without
         # transitioning to a specific python version.
-        default_version = _major_minor_version(DEFAULT_PYTHON_VERSION)
         if default_version == major_minor:
             platform_config_setting_name = "is_{}_{}".format(
                 platforms[0].os,
@@ -195,13 +188,7 @@ def _create_whl_repos(*, module_ctx, pip_attr, whl_map, whl_overrides, config_se
             )
             platform_config_settings.append("//:" + platform_config_setting_name)
 
-        target_platforms = [
-            "cp{}_{}_{}".format(
-                version_label(major_minor),
-                platforms[0].os,
-                platforms[0].cpu,
-            ),
-        ]
+        target_platforms = ["cp{}_{}_{}".format(version_label(major_minor), platforms[0].os, platforms[0].cpu)]
     else:
         platform_config_setting_name = "is_python_{}".format(major_minor)
 
@@ -214,6 +201,9 @@ def _create_whl_repos(*, module_ctx, pip_attr, whl_map, whl_overrides, config_se
             visibility = ["//:__subpackages__"],
         )
         platform_config_settings.append("//:" + platform_config_setting_name)
+        if default_version == major_minor:
+            platform_config_settings.append("//conditions:default")
+            config_settings_map[hub_name]["//conditions:default"] = "//conditions:default"
         requirements_lock = locked_requirements_label(module_ctx, pip_attr)
 
     # Parse the requirements file directly in starlark to get the information
@@ -261,7 +251,7 @@ def _create_whl_repos(*, module_ctx, pip_attr, whl_map, whl_overrides, config_se
         for whl_name in group_whls
     }
 
-    group_repo = "%s__groups" % (pip_name,)
+    group_repo = "%s__%s__groups" % (pip_name, pip_attr.experimental_whl_platform)
     group_library(
         name = group_repo,
         repo_prefix = pip_name + "_",
@@ -279,17 +269,24 @@ def _create_whl_repos(*, module_ctx, pip_attr, whl_map, whl_overrides, config_se
         group_deps = requirement_cycles.get(group_name, [])
 
         repo_name = "{}_{}".format(pip_name, whl_name)
+        if pip_attr.experimental_whl_platform:
+            repo_name = "{}__{}".format(repo_name, pip_attr.experimental_whl_platform)
+            whl_pip_name = "{}_{}".format(pip_name, pip_attr.experimental_whl_platform)
+        else:
+            whl_pip_name = pip_name
+
         whl_library(
             name = repo_name,
             requirement = requirement_line,
-            repo = pip_name,
-            repo_prefix = pip_name + "_",
+            repo = whl_pip_name,
+            repo_prefix = whl_pip_name + "_",
             annotation = annotation,
             whl_patches = {
                 p: json.encode(args)
                 for p, args in whl_overrides.get(whl_name, {}).items()
             },
             experimental_target_platforms = target_platforms,
+            experimental_use_hub_for_deps = hub_name,
             python_interpreter = pip_attr.python_interpreter,
             python_interpreter_target = python_interpreter_target,
             quiet = pip_attr.quiet,
@@ -481,7 +478,14 @@ def _pip_impl(module_ctx):
 
     for hub_name, whl_map in hub_whl_map.items():
         build_file_contents = BUILD_FILE_CONTENTS
-        config_settings = config_settings_map.get(hub_name, {}).values()
+        has_default = False
+        config_settings = []
+        for value in config_settings_map.get(hub_name, {}).values():
+            if value == "//conditions:default":
+                has_default = True
+                continue
+            config_settings.append(value)
+
         if config_settings:
             self, _, _ = str(Label("//:MODULE.bazel")).partition("//")
             build_file_contents = "\n".join([
@@ -501,6 +505,7 @@ def _pip_impl(module_ctx):
                 for key, value in whl_map.items()
             },
             default_version = _major_minor_version(DEFAULT_PYTHON_VERSION),
+            render_default_version = not has_default,
         )
 
 def _pip_parse_ext_attrs():

python/private/bzlmod/pip_repository.bzl

@@ -31,7 +31,7 @@ def _pip_repository_impl(rctx):
             key: [whl_alias(**v) for v in json.decode(values)]
             for key, values in rctx.attr.whl_map.items()
         },
-        default_version = rctx.attr.default_version,
+        default_version = rctx.attr.default_version if rctx.attr.render_default_version else None,
     )
     for path, contents in aliases.items():
         rctx.file(path, contents)
@@ -72,6 +72,9 @@ This is the default python version in the format of X.Y. This should match
 what is setup by the 'python' extension using the 'is_default = True'
 setting.""",
     ),
+    "render_default_version": attr.bool(
+        default = True,
+    ),
     "repo_name": attr.string(
         mandatory = True,
         doc = "The apparent name of the repo. This is needed because in bzlmod, the name attribute becomes the canonical name.",

python/private/render_pkg_aliases.bzl

@@ -57,7 +57,7 @@ def _render_whl_library_alias(
     selects = {}
     no_match_error = "_NO_MATCH_ERROR"
     default = None
-    for alias in sorted(aliases, key = lambda x: x.version):
+    for alias in sorted(aliases, key = lambda x: x.config_setting):
         actual = "@{repo}//:{name}".format(repo = alias.repo, name = name)
         selects[alias.config_setting] = actual
         if alias.version == default_version:
@@ -81,8 +81,12 @@ def _render_common_aliases(*, name, aliases, default_version = None):
     ]
 
     versions = None
+    has_default = False
     if aliases:
-        versions = sorted([v.version for v in aliases if v.version])
+        versions = sorted([a.version for a in aliases if a.version])
+        has_default = len([a.config_setting for a in aliases if a.config_setting == "//conditions:default"]) == 1
+        if has_default:
+            default_version = None
 
     if not versions or default_version in versions:
         pass