changes to pip_repository source files now re-trigger the repo rule (#601)

* changes to pip_repository source files now re-trigger the repo rule

* update diff test message.

* added note about auto-gen.

* Updated docs

* remove unnecessary snippet

* installer -> updater

Co-authored-by: Jonathon Belotti <[email protected]>

Author: UebelAndre

docs/BUILD

@@ -61,9 +61,7 @@ bzl_library(
 bzl_library(
     name = "pip_install_bzl",
     srcs = [
-        "//python/pip_install:pip_repository.bzl",
-        "//python/pip_install:repositories.bzl",
-        "//python/pip_install:requirements.bzl",
+        "//python/pip_install:bzl",
     ],
     deps = [
         ":defs",

python/BUILD

@@ -46,6 +46,7 @@ filegroup(
         "defs.bzl",
         "packaging.bzl",
         "pip.bzl",
+        "//python/pip_install:bzl",
         "//python/private:bzl",
     ],
     visibility = ["//:__pkg__"],

python/pip_install/BUILD

@@ -7,14 +7,26 @@ filegroup(
         "pip_compile.py",
         "//python/pip_install/extract_wheels:distribution",
         "//python/pip_install/parse_requirements_to_bzl:distribution",
+        "//python/pip_install/private:distribution",
     ],
     visibility = ["//:__pkg__"],
 )
 
 filegroup(
     name = "bzl",
-    srcs = glob(["*.bzl"]),
-    visibility = ["//:__pkg__"],
+    srcs = glob(["*.bzl"]) + [
+        "//python/pip_install/private:bzl_srcs",
+    ],
+    visibility = ["//:__subpackages__"],
+)
+
+filegroup(
+    name = "py_srcs",
+    srcs = [
+        "//python/pip_install/extract_wheels:py_srcs",
+        "//python/pip_install/parse_requirements_to_bzl:py_srcs",
+    ],
+    visibility = ["//visibility:public"],
 )
 
 exports_files(

python/pip_install/extract_wheels/BUILD

@@ -17,3 +17,14 @@ filegroup(
     ],
     visibility = ["//python/pip_install:__subpackages__"],
 )
+
+filegroup(
+    name = "py_srcs",
+    srcs = glob(
+        include = ["**/*.py"],
+        exclude = ["**/*_test.py"],
+    ) + [
+        "//python/pip_install/extract_wheels/lib:py_srcs",
+    ],
+    visibility = ["//python/pip_install:__subpackages__"],
+)

python/pip_install/extract_wheels/lib/BUILD

@@ -110,3 +110,12 @@ filegroup(
     ),
     visibility = ["//python/pip_install:__subpackages__"],
 )
+
+filegroup(
+    name = "py_srcs",
+    srcs = glob(
+        include = ["**/*.py"],
+        exclude = ["**/*_test.py"],
+    ),
+    visibility = ["//python/pip_install:__subpackages__"],
+)

python/pip_install/parse_requirements_to_bzl/BUILD

@@ -41,3 +41,14 @@ filegroup(
     ],
     visibility = ["//python/pip_install:__subpackages__"],
 )
+
+filegroup(
+    name = "py_srcs",
+    srcs = glob(
+        include = ["**/*.py"],
+        exclude = ["**/*_test.py"],
+    ) + [
+        "//python/pip_install/parse_requirements_to_bzl/extract_single_wheel:py_srcs",
+    ],
+    visibility = ["//python/pip_install:__subpackages__"],
+)

python/pip_install/parse_requirements_to_bzl/extract_single_wheel/BUILD

@@ -6,3 +6,12 @@ filegroup(
     ),
     visibility = ["//python/pip_install:__subpackages__"],
 )
+
+filegroup(
+    name = "py_srcs",
+    srcs = glob(
+        include = ["**/*.py"],
+        exclude = ["**/*_test.py"],
+    ),
+    visibility = ["//python/pip_install:__subpackages__"],
+)

python/pip_install/pip_repository.bzl

@@ -1,6 +1,7 @@
 ""
 
 load("//python/pip_install:repositories.bzl", "all_requirements")
+load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS")
 
 def _construct_pypath(rctx):
     """Helper function to construct a PYTHONPATH.
@@ -250,6 +251,11 @@ For incremental mode the packages will be of the form
         default = 600,
         doc = "Timeout (in seconds) on the rule's execution duration.",
     ),
+    "_py_srcs": attr.label_list(
+        doc = "Python sources used in the repository rule",
+        allow_files = True,
+        default = PIP_INSTALL_PY_SRCS,
+    ),
 }
 
 pip_repository_attrs = {

python/pip_install/private/BUILD

@@ -0,0 +1,24 @@
+load(":pip_install_utils.bzl", "srcs_module")
+
+package(default_visibility = ["//:__subpackages__"])
+
+exports_files([
+    "srcs.bzl",
+])
+
+filegroup(
+    name = "distribution",
+    srcs = glob(["*"]),
+    visibility = ["//python/pip_install:__subpackages__"],
+)
+
+filegroup(
+    name = "bzl_srcs",
+    srcs = glob(["*.bzl"]),
+)
+
+srcs_module(
+    name = "srcs_module",
+    srcs = "//python/pip_install:py_srcs",
+    dest = ":srcs.bzl",
+)

python/pip_install/private/pip_install_utils.bzl

@@ -0,0 +1,118 @@
+"""Utilities for `rules_python` pip rules"""
+
+_SRCS_TEMPLATE = """\
+\"\"\"A generate file containing all source files used for `@rules_python//python/pip_install:pip_repository.bzl` rules
+
+This file is auto-generated from the `@rules_python//python/pip_install/private:srcs_module.install` target. Please
+`bazel run` this target to apply any updates. Note that doing so will discard any local modifications.
+"\"\"
+
+# Each source file is tracked as a target so `pip_repository` rules will know to automatically rebuild if any of the
+# sources changed.
+PIP_INSTALL_PY_SRCS = [
+    {srcs}
+]
+"""
+
+def _src_label(file):
+    dir_path, file_name = file.short_path.rsplit("/", 1)
+
+    return "@rules_python//{}:{}".format(
+        dir_path,
+        file_name,
+    )
+
+def _srcs_module_impl(ctx):
+    srcs = [_src_label(src) for src in ctx.files.srcs]
+    if not srcs:
+        fail("`srcs` cannot be empty")
+    output = ctx.actions.declare_file(ctx.label.name)
+
+    ctx.actions.write(
+        output = output,
+        content = _SRCS_TEMPLATE.format(
+            srcs = "\n    ".join(["\"{}\",".format(src) for src in srcs]),
+        ),
+    )
+
+    return DefaultInfo(
+        files = depset([output]),
+    )
+
+_srcs_module = rule(
+    doc = "A rule for writing a list of sources to a templated file",
+    implementation = _srcs_module_impl,
+    attrs = {
+        "srcs": attr.label(
+            doc = "A filegroup of source files",
+            allow_files = True,
+        ),
+    },
+)
+
+_INSTALLER_TEMPLATE = """\
+#!/bin/bash
+set -euo pipefail
+cp -f "{path}" "${{BUILD_WORKSPACE_DIRECTORY}}/{dest}"
+"""
+
+def _srcs_updater_impl(ctx):
+    output = ctx.actions.declare_file(ctx.label.name + ".sh")
+    target_file = ctx.file.input
+    dest = ctx.file.dest.short_path
+
+    ctx.actions.write(
+        output = output,
+        content = _INSTALLER_TEMPLATE.format(
+            path = target_file.short_path,
+            dest = dest,
+        ),
+        is_executable = True,
+    )
+
+    return DefaultInfo(
+        files = depset([output]),
+        runfiles = ctx.runfiles(files = [target_file]),
+        executable = output,
+    )
+
+_srcs_updater = rule(
+    doc = "A rule for writing a `srcs.bzl` file back to the repository",
+    implementation = _srcs_updater_impl,
+    attrs = {
+        "dest": attr.label(
+            doc = "The target file to write the new `input` to.",
+            allow_single_file = ["srcs.bzl"],
+            mandatory = True,
+        ),
+        "input": attr.label(
+            doc = "The file to write back to the repository",
+            allow_single_file = True,
+            mandatory = True,
+        ),
+    },
+    executable = True,
+)
+
+def srcs_module(name, dest, **kwargs):
+    """A helper rule to ensure `pip_repository` rules are always up to date
+
+    Args:
+        name (str): The name of the sources module
+        dest (str): The filename the module should be written as in the current package.
+        **kwargs (dict): Additional keyword arguments
+    """
+    tags = kwargs.pop("tags", [])
+
+    _srcs_module(
+        name = name,
+        tags = tags,
+        **kwargs
+    )
+
+    _srcs_updater(
+        name = name + ".update",
+        input = name,
+        dest = dest,
+        tags = tags,
+    )