Make the pip extension reproducible if PyPI is not called

Author: aignas

CHANGELOG.md

@@ -35,7 +35,11 @@ A brief description of the categories of changes:
   Implemented the use of `pypi/stdlib-list` for standard library module verification.
 * (pip.parse): Do not ignore yanked packages when using `experimental_index_url`.
   This is to mimic what `uv` is doing. We will print a warning instead.
-* (pip.parse): Add references to all supported wheels when using `experimental_index_url`.
+* (pip.parse): Add references to all supported wheels when using `experimental_index_url`
+  to allowing to correctly fetch the wheels for the right platform. See the
+  updated docs on how to use the feature. This is work towards addressing
+  [#735](https://github.com/bazelbuild/rules_python/issues/735) and
+  [#260](https://github.com/bazelbuild/rules_python/issues/260).
 
 ### Fixed
 * (gazelle) Remove `visibility` from `NonEmptyAttr`.
@@ -53,6 +57,12 @@ A brief description of the categories of changes:
   replaced by whl_modifications.
 * (pip) Correctly select wheels when the python tag includes minor versions.
   See ([#1930](https://github.com/bazelbuild/rules_python/issues/1930))
+* (pip.parse): The lock file is now reproducible on any host platform if the
+  `experimental_index_url` is not used by any of the modules in the dependency
+  chain. To make the lock file identical on each `os` and `arch`, please use
+  the `experimental_index_url` feature which will fetch metadata from PyPI or a
+  different private index and write the contents to the lock file. Fixes
+  [#1643](https://github.com/bazelbuild/rules_python/issues/1643).
 
 ### Added
 * (rules) Precompiling Python source at build time is available. but is

MODULE.bazel

@@ -56,9 +56,10 @@ register_toolchains("@pythons_hub//:all")
 #####################
 # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.
 
-pip = use_extension("//python/extensions:pip.bzl", "pip")
+pip = use_extension("//python/private/bzlmod:pip.bzl", "pip_internal")
 pip.parse(
-    experimental_index_url = "https://pypi.org/simple",
+    envsubst = ["PIP_INDEX_URL"],
+    experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
     hub_name = "rules_python_publish_deps",
     python_version = "3.11",
     requirements_by_platform = {
@@ -80,8 +81,8 @@ bazel_dep(name = "rules_go", version = "0.41.0", dev_dependency = True, repo_nam
 bazel_dep(name = "gazelle", version = "0.33.0", dev_dependency = True, repo_name = "bazel_gazelle")
 
 dev_pip = use_extension(
-    "//python/extensions:pip.bzl",
-    "pip",
+    "//python/private/bzlmod:pip.bzl",
+    "pip_internal",
     dev_dependency = True,
 )
 dev_pip.parse(

python/private/bzlmod/pip.bzl

@@ -103,6 +103,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
 def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, simpleapi_cache):
     logger = repo_utils.logger(module_ctx)
     python_interpreter_target = pip_attr.python_interpreter_target
+    is_hub_reproducible = True
 
     # if we do not have the python_interpreter set in the attributes
     # we programmatically find it.
@@ -240,7 +241,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s
         )
         whl_library_args.update({k: v for k, (v, default) in maybe_args_with_default.items() if v == default})
 
-        if pip_attr.experimental_index_url:
+        if get_index_urls:
             # TODO @aignas 2024-05-26: move to a separate function
             found_something = False
             for requirement in requirements:
@@ -250,6 +251,8 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s
                         continue
 
                     found_something = True
+                    is_hub_reproducible = False
+
                     if pip_attr.netrc:
                         whl_library_args["netrc"] = pip_attr.netrc
                     if pip_attr.auth_patterns:
@@ -300,7 +303,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s
             # packages they want to download, in that case there will be always
             # a requirement here, so we will not be in this code branch.
             continue
-        elif pip_attr.experimental_index_url:
+        elif get_index_urls:
             logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
 
         whl_library_args["requirement"] = requirement.requirement_line
@@ -318,6 +321,8 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s
             ),
         )
 
+    return is_hub_reproducible
+
 def _pip_impl(module_ctx):
     """Implementation of a class tag that creates the pip hub and corresponding pip spoke whl repositories.
 
@@ -427,6 +432,7 @@ def _pip_impl(module_ctx):
     hub_group_map = {}
 
     simpleapi_cache = {}
+    is_extension_reproducible = True
 
     for mod in module_ctx.modules:
         for pip_attr in mod.tags.parse:
@@ -463,7 +469,8 @@ def _pip_impl(module_ctx):
             else:
                 pip_hub_map[pip_attr.hub_name].python_versions.append(pip_attr.python_version)
 
-            _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
+            is_hub_reproducible = _create_whl_repos(module_ctx, pip_attr, hub_whl_map, whl_overrides, hub_group_map, simpleapi_cache)
+            is_extension_reproducible = is_extension_reproducible and is_hub_reproducible
 
     for hub_name, whl_map in hub_whl_map.items():
         pip_repository(
@@ -477,6 +484,20 @@ def _pip_impl(module_ctx):
             groups = hub_group_map.get(hub_name),
         )
 
+    if bazel_features.external_deps.extension_metadata_has_reproducible:
+        return module_ctx.extension_metadata(reproducible = is_extension_reproducible)
+    else:
+        return None
+
+def _pip_non_reproducible(module_ctx):
+    _pip_impl(module_ctx)
+
+    if bazel_features.external_deps.extension_metadata_has_reproducible:
+        # We allow for calling the PyPI index and that will go into the MODULE.bazel.lock file
+        return module_ctx.extension_metadata(reproducible = False)
+    else:
+        return None
+
 def _pip_parse_ext_attrs():
     attrs = dict({
         "experimental_extra_index_urls": attr.string_list(
@@ -492,6 +513,7 @@ This is equivalent to `--extra-index-urls` `pip` option.
             default = [],
         ),
         "experimental_index_url": attr.string(
+            default = "https://pypi.org/simple",
             doc = """\
 The index URL to use for downloading wheels using bazel downloader. This value is going
 to be subject to `envsubst` substitutions if necessary.
@@ -676,17 +698,6 @@ Apply any overrides (e.g. patches) to a given Python distribution defined by
 other tags in this extension.""",
 )
 
-def _extension_extra_args():
-    args = {}
-
-    if bazel_features.external_deps.module_extension_has_os_arch_dependent:
-        args = args | {
-            "arch_dependent": True,
-            "os_dependent": True,
-        }
-
-    return args
-
 pip = module_extension(
     doc = """\
 This extension is used to make dependencies from pip available.
@@ -729,7 +740,54 @@ extension.
 """,
         ),
     },
-    **_extension_extra_args()
+)
+
+pip_internal = module_extension(
+    doc = """\
+This extension is used to make dependencies from pypi available.
+
+For now this is intended to be used internally so that usage of the `pip`
+extension in `rules_python` does not affect the evaluations of the extension
+for the consumers.
+
+pip.parse:
+To use, call `pip.parse()` and specify `hub_name` and your requirements file.
+Dependencies will be downloaded and made available in a repo named after the
+`hub_name` argument.
+
+Each `pip.parse()` call configures a particular Python version. Multiple calls
+can be made to configure different Python versions, and will be grouped by
+the `hub_name` argument. This allows the same logical name, e.g. `@pypi//numpy`
+to automatically resolve to different, Python version-specific, libraries.
+
+pip.whl_mods:
+This tag class is used to help create JSON files to describe modifications to
+the BUILD files for wheels.
+""",
+    implementation = _pip_non_reproducible,
+    tag_classes = {
+        "override": _override_tag,
+        "parse": tag_class(
+            attrs = _pip_parse_ext_attrs(),
+            doc = """\
+This tag class is used to create a pypi hub and all of the spokes that are part of that hub.
+This tag class reuses most of the pypi attributes that are found in
+@rules_python//python/pip_install:pip_repository.bzl.
+The exception is it does not use the arg 'repo_prefix'.  We set the repository
+prefix for the user and the alias arg is always True in bzlmod.
+""",
+        ),
+        "whl_mods": tag_class(
+            attrs = _whl_mod_attrs(),
+            doc = """\
+This tag class is used to create JSON file that are used when calling wheel_builder.py.  These
+JSON files contain instructions on how to modify a wheel's project.  Each of the attributes
+create different modifications based on the type of attribute. Previously to bzlmod these
+JSON files where referred to as annotations, and were renamed to whl_modifications in this
+extension.
+""",
+        ),
+    },
 )
 
 def _whl_mods_repo_impl(rctx):