fix(pip.parse): make the pip extension reproducible if PyPI is not called (#1937)

With this PR we can finally have fewer lock file entries in setups which
do not use the `experimental_index_url` feature yet. This is fully
backwards compatible change as it relies on `bazel` doing the right
thing and regenerating the lock file.

Fixes #1643.

Author: aignas

CHANGELOG.md

@@ -57,6 +57,12 @@ A brief description of the categories of changes:
   replaced by whl_modifications.
 * (pip) Correctly select wheels when the python tag includes minor versions.
   See ([#1930](https://github.com/bazelbuild/rules_python/issues/1930))
+* (pip.parse): The lock file is now reproducible on any host platform if the
+  `experimental_index_url` is not used by any of the modules in the dependency
+  chain. To make the lock file identical on each `os` and `arch`, please use
+  the `experimental_index_url` feature which will fetch metadata from PyPI or a
+  different private index and write the contents to the lock file. Fixes
+  [#1643](https://github.com/bazelbuild/rules_python/issues/1643).
 
 ### Added
 * (rules) Precompiling Python source at build time is available. but is

MODULE.bazel

@@ -56,9 +56,8 @@ register_toolchains("@pythons_hub//:all")
 #####################
 # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.
 
-pip = use_extension("//python/extensions:pip.bzl", "pip")
+pip = use_extension("//python/private/bzlmod:pip.bzl", "pip_internal")
 pip.parse(
-    experimental_index_url = "https://pypi.org/simple",
     hub_name = "rules_python_publish_deps",
     python_version = "3.11",
     requirements_by_platform = {
@@ -80,13 +79,11 @@ bazel_dep(name = "rules_go", version = "0.41.0", dev_dependency = True, repo_nam
 bazel_dep(name = "gazelle", version = "0.33.0", dev_dependency = True, repo_name = "bazel_gazelle")
 
 dev_pip = use_extension(
-    "//python/extensions:pip.bzl",
-    "pip",
+    "//python/private/bzlmod:pip.bzl",
+    "pip_internal",
     dev_dependency = True,
 )
 dev_pip.parse(
-    envsubst = ["PIP_INDEX_URL"],
-    experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
     experimental_requirement_cycles = {
         "sphinx": [
             "sphinx",
@@ -102,8 +99,6 @@ dev_pip.parse(
     requirements_lock = "//docs/sphinx:requirements.txt",
 )
 dev_pip.parse(
-    envsubst = ["PIP_INDEX_URL"],
-    experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
     hub_name = "pypiserver",
     python_version = "3.11",
     requirements_lock = "//examples/wheel:requirements_server.txt",

python/private/bzlmod/pip.bzl

@@ -103,6 +103,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
 def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache):
     logger = repo_utils.logger(module_ctx)
     python_interpreter_target = pip_attr.python_interpreter_target
+    is_hub_reproducible = True
 
     # if we do not have the python_interpreter set in the attributes
     # we programmatically find it.
@@ -274,6 +275,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s
             logger.debug(lambda: "Selected: {}".format(distribution))
 
             if distribution:
+                is_hub_reproducible = False
                 whl_library_args["requirement"] = requirement.srcs.requirement
                 whl_library_args["urls"] = [distribution.url]
                 whl_library_args["sha256"] = distribution.sha256
@@ -303,6 +305,8 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s
             ),
         )
 
+    return is_hub_reproducible
+
 def _pip_impl(module_ctx):
     """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.
 
@@ -412,6 +416,7 @@ def _pip_impl(module_ctx):
     hub_group_map = {}
 
     simpleapi_cache = {}
+    is_extension_reproducible = True
 
     for mod in module_ctx.modules:
         for pip_attr in mod.tags.parse:
@@ -448,7 +453,8 @@ def _pip_impl(module_ctx):
             else:
                 pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)
 
-            _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
+            is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
+            is_extension_reproducible = is_extension_reproducible and is_hub_reproducible
 
     for hub_name, whl_map in hub_whl_map.items():
         pip_repository(
@@ -462,7 +468,34 @@ def _pip_impl(module_ctx):
             groups = hub_group_map.get(hub_name),
         )
 
-def _pip_parse_ext_attrs():
+    if bazel_features.external_deps.extension_metadata_has_reproducible:
+        # If we are not using the `experimental_index_url feature, the extension is fully
+        # deterministic and we don't need to create a lock entry for it.
+        #
+        # In order to be able to dogfood the `experimental_index_url` feature before it gets
+        # stabilized, we have created the `_pip_non_reproducible` function, that will result
+        # in extra entries in the lock file.
+        return module_ctx.extension_metadata(reproducible = is_extension_reproducible)
+    else:
+        return None
+
+def _pip_non_reproducible(module_ctx):
+    _pip_impl(module_ctx)
+
+    # We default to calling the PyPI index and that will go into the
+    # MODULE.bazel.lock file, hence return nothing here.
+    return None
+
+def _pip_parse_ext_attrs(**kwargs):
+    """Get the attributes for the pip extension.
+
+    Args:
+        **kwargs: A kwarg for setting defaults for the specific attributes. The
+        key is expected to be the same as the attribute key.
+
+    Returns:
+        A dict of attributes.
+    """
     attrs = dict({
         "experimental_extra_index_urls": attr.string_list(
             doc = """\
@@ -477,6 +510,7 @@ This is equivalent to `--extra-index-urls` `pip` option.
             default = [],
         ),
         "experimental_index_url": attr.string(
+            default = kwargs.get("experimental_index_url", ""),
             doc = """\
 The index URL to use for downloading wheels using bazel downloader. This value is going
 to be subject to `envsubst` substitutions if necessary.
@@ -661,17 +695,6 @@ Apply any overrides (e.g. patches) to a given Python distribution defined by
 other tags in this extension.""",
 )
 
-def _extension_extra_args():
-    args = {}
-
-    if bazel_features.external_deps.module_extension_has_os_arch_dependent:
-        args = args | {
-            "arch_dependent": True,
-            "os_dependent": True,
-        }
-
-    return args
-
 pip = module_extension(
     doc = """\
 This extension is used to make dependencies from pip available.
@@ -714,7 +737,56 @@ extension.
 """,
         ),
     },
-    **_extension_extra_args()
+)
+
+pip_internal = module_extension(
+    doc = """\
+This extension is used to make dependencies from pypi available.
+
+For now this is intended to be used internally so that usage of the `pip`
+extension in `rules_python` does not affect the evaluations of the extension
+for the consumers.
+
+pip.parse:
+To use, call `pip.parse()` and specify `hub_name` and your requirements file.
+Dependencies will be downloaded and made available in a repo named after the
+`hub_name` argument.
+
+Each `pip.parse()` call configures a particular Python version. Multiple calls
+can be made to configure different Python versions, and will be grouped by
+the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy`
+to automatically resolve to different, Python version-specific, libraries.
+
+pip.whl_mods:
+This tag class is used to help create JSON files to describe modifications to
+the BUILD files for wheels.
+""",
+    implementation = _pip_non_reproducible,
+    tag_classes = {
+        "override": _override_tag,
+        "parse": tag_class(
+            attrs = _pip_parse_ext_attrs(
+                experimental_index_url = "https://pypi.org/simple",
+            ),
+            doc = """\
+This tag class is used to create a pypi hub and all of the spokes that are part of that hub.
+This tag class reuses most of the pypi attributes that are found in
+@rules_python//python/pip_install:pip_repository.bzl.
+The exception is it does not use the arg 'repo_prefix'.  We set the repository
+prefix for the user and the alias arg is always True in bzlmod.
+""",
+        ),
+        "whl_mods": tag_class(
+            attrs = _whl_mod_attrs(),
+            doc = """\
+This tag class is used to create JSON file that are used when calling wheel_builder.py.  These
+JSON files contain instructions on how to modify a wheel's project.  Each of the attributes
+create different modifications based on the type of attribute. Previously to bzlmod these
+JSON files where referred to as annotations, and were renamed to whl_modifications in this
+extension.
+""",
+        ),
+    },
 )
 
 def _whl_mods_repo_impl(rctx):