feat: bzl file per rule/provider (#1122)

This is basically a performance optimization. Bazel can only see that a
bzl file changed, not the particular contents. This means that any
downstream bzl file loading it is invalidated, even if it doesn't load
any of the affected code. As an example, if a package only loads
`py_library.bzl`, then changing `py_test.bzl` doesn't need to invalidate
all libraries.

* Also removes some more extraneous license comments

Work towards #1069

Author: rickeylev

docs/BUILD.bazel

@@ -56,7 +56,11 @@ bzl_library(
         "//python:defs.bzl",
         "//python/private:reexports.bzl",
     ],
-    deps = [":bazel_python_tools"],
+    deps = [
+        ":bazel_python_tools",
+        "//python:defs_bzl",
+        "//python/private:reexports_bzl",
+    ],
 )
 
 bzl_library(

python/BUILD.bazel

@@ -24,11 +24,11 @@ that @rules_python//python is only concerned with the core rules.
 """
 
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
-load(":defs.bzl", "current_py_toolchain")
+load(":current_py_toolchain.bzl", "current_py_toolchain")
 
 package(default_visibility = ["//visibility:public"])
 
-licenses(["notice"])  # Apache 2.0
+licenses(["notice"])
 
 filegroup(
     name = "distribution",
@@ -41,13 +41,22 @@ filegroup(
     visibility = ["//:__pkg__"],
 )
 
+# ========= bzl_library targets end =========
+
+bzl_library(
+    name = "current_py_toolchain_bzl",
+    srcs = ["current_py_toolchain.bzl"],
+)
+
 bzl_library(
     name = "defs_bzl",
     srcs = [
         "defs.bzl",
     ],
     visibility = ["//visibility:public"],
     deps = [
+        ":current_py_toolchain_bzl",
+        ":py_import_bzl",
         "//python/private:bazel_tools_bzl",
         "//python/private:reexports_bzl",
     ],
@@ -64,6 +73,57 @@ bzl_library(
     ],
 )
 
+bzl_library(
+    name = "py_binary_bzl",
+    srcs = ["py_binary.bzl"],
+    deps = ["//python/private:reexports_bzl"],
+)
+
+bzl_library(
+    name = "py_import_bzl",
+    srcs = ["py_import.bzl"],
+    deps = [":py_info_bzl"],
+)
+
+bzl_library(
+    name = "py_info_bzl",
+    srcs = ["py_info.bzl"],
+    deps = ["//python/private:reexports_bzl"],
+)
+
+bzl_library(
+    name = "py_library_bzl",
+    srcs = ["py_library.bzl"],
+    deps = ["//python/private:reexports_bzl"],
+)
+
+bzl_library(
+    name = "py_runtime_bzl",
+    srcs = ["py_runtime.bzl"],
+    deps = ["//python/private:reexports_bzl"],
+)
+
+bzl_library(
+    name = "py_runtime_pair_bzl",
+    srcs = ["py_runtime_pair.bzl"],
+    deps = ["//python/private:reexports_bzl"],
+)
+
+bzl_library(
+    name = "py_runtime_info_bzl",
+    srcs = ["py_runtime_info.bzl"],
+    deps = ["//python/private:reexports_bzl"],
+)
+
+bzl_library(
+    name = "py_test_bzl",
+    srcs = ["py_test.bzl"],
+    deps = ["//python/private:reexports_bzl"],
+)
+
+# NOTE: Remember to add bzl_library targets to //tests:bzl_libraries
+# ========= bzl_library targets end =========
+
 # Filegroup of bzl files that can be used by downstream rules for documentation generation
 filegroup(
     name = "bzl",

python/current_py_toolchain.bzl

@@ -0,0 +1,58 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Public entry point for current_py_toolchain rule."""
+
+def _current_py_toolchain_impl(ctx):
+    toolchain = ctx.toolchains[ctx.attr._toolchain]
+
+    direct = []
+    transitive = []
+    vars = {}
+
+    if toolchain.py3_runtime and toolchain.py3_runtime.interpreter:
+        direct.append(toolchain.py3_runtime.interpreter)
+        transitive.append(toolchain.py3_runtime.files)
+        vars["PYTHON3"] = toolchain.py3_runtime.interpreter.path
+
+    if toolchain.py2_runtime and toolchain.py2_runtime.interpreter:
+        direct.append(toolchain.py2_runtime.interpreter)
+        transitive.append(toolchain.py2_runtime.files)
+        vars["PYTHON2"] = toolchain.py2_runtime.interpreter.path
+
+    files = depset(direct, transitive = transitive)
+    return [
+        toolchain,
+        platform_common.TemplateVariableInfo(vars),
+        DefaultInfo(
+            runfiles = ctx.runfiles(transitive_files = files),
+            files = files,
+        ),
+    ]
+
+current_py_toolchain = rule(
+    doc = """
+    This rule exists so that the current python toolchain can be used in the `toolchains` attribute of
+    other rules, such as genrule. It allows exposing a python toolchain after toolchain resolution has
+    happened, to a rule which expects a concrete implementation of a toolchain, rather than a
+    toolchain_type which could be resolved to that toolchain.
+    """,
+    implementation = _current_py_toolchain_impl,
+    attrs = {
+        "_toolchain": attr.string(default = str(Label("@bazel_tools//tools/python:toolchain_type"))),
+    },
+    toolchains = [
+        str(Label("@bazel_tools//tools/python:toolchain_type")),
+    ],
+)

python/defs.bzl

@@ -27,105 +27,18 @@ load(
     _py_runtime_pair = "py_runtime_pair",
     _py_test = "py_test",
 )
+load(":current_py_toolchain.bzl", _current_py_toolchain = "current_py_toolchain")
+load(":py_import.bzl", _py_import = "py_import")
 
 # Exports of native-defined providers.
 
 PyInfo = internal_PyInfo
 
 PyRuntimeInfo = internal_PyRuntimeInfo
 
-def _current_py_toolchain_impl(ctx):
-    toolchain = ctx.toolchains[ctx.attr._toolchain]
+current_py_toolchain = _current_py_toolchain
 
-    direct = []
-    transitive = []
-    vars = {}
-
-    if toolchain.py3_runtime and toolchain.py3_runtime.interpreter:
-        direct.append(toolchain.py3_runtime.interpreter)
-        transitive.append(toolchain.py3_runtime.files)
-        vars["PYTHON3"] = toolchain.py3_runtime.interpreter.path
-
-    if toolchain.py2_runtime and toolchain.py2_runtime.interpreter:
-        direct.append(toolchain.py2_runtime.interpreter)
-        transitive.append(toolchain.py2_runtime.files)
-        vars["PYTHON2"] = toolchain.py2_runtime.interpreter.path
-
-    files = depset(direct, transitive = transitive)
-    return [
-        toolchain,
-        platform_common.TemplateVariableInfo(vars),
-        DefaultInfo(
-            runfiles = ctx.runfiles(transitive_files = files),
-            files = files,
-        ),
-    ]
-
-current_py_toolchain = rule(
-    doc = """
-    This rule exists so that the current python toolchain can be used in the `toolchains` attribute of
-    other rules, such as genrule. It allows exposing a python toolchain after toolchain resolution has
-    happened, to a rule which expects a concrete implementation of a toolchain, rather than a
-    toolchain_type which could be resolved to that toolchain.
-    """,
-    implementation = _current_py_toolchain_impl,
-    attrs = {
-        "_toolchain": attr.string(default = str(Label("@bazel_tools//tools/python:toolchain_type"))),
-    },
-    toolchains = [
-        str(Label("@bazel_tools//tools/python:toolchain_type")),
-    ],
-)
-
-def _py_import_impl(ctx):
-    # See https://github.com/bazelbuild/bazel/blob/0.24.0/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java#L104 .
-    import_paths = [
-        "/".join([ctx.workspace_name, x.short_path])
-        for x in ctx.files.srcs
-    ]
-
-    return [
-        DefaultInfo(
-            default_runfiles = ctx.runfiles(ctx.files.srcs, collect_default = True),
-        ),
-        PyInfo(
-            transitive_sources = depset(transitive = [
-                dep[PyInfo].transitive_sources
-                for dep in ctx.attr.deps
-            ]),
-            imports = depset(direct = import_paths, transitive = [
-                dep[PyInfo].imports
-                for dep in ctx.attr.deps
-            ]),
-        ),
-    ]
-
-py_import = rule(
-    doc = """This rule allows the use of Python packages as dependencies.
-
-    It imports the given `.egg` file(s), which might be checked in source files,
-    fetched externally as with `http_file`, or produced as outputs of other rules.
-
-    It may be used like a `py_library`, in the `deps` of other Python rules.
-
-    This is similar to [java_import](https://docs.bazel.build/versions/master/be/java.html#java_import).
-    """,
-    implementation = _py_import_impl,
-    attrs = {
-        "deps": attr.label_list(
-            doc = "The list of other libraries to be linked in to the " +
-                  "binary target.",
-            providers = [PyInfo],
-        ),
-        "srcs": attr.label_list(
-            doc = "The list of Python package files provided to Python targets " +
-                  "that depend on this target. Note that currently only the .egg " +
-                  "format is accepted. For .whl files, try the whl_library rule. " +
-                  "We accept contributions to extend py_import to handle .whl.",
-            allow_files = [".egg"],
-        ),
-    },
-)
+py_import = _py_import
 
 # Re-exports of Starlark-defined symbols in @bazel_tools//tools/python.
 

python/private/BUILD.bazel

@@ -34,7 +34,10 @@ filegroup(
 bzl_library(
     name = "reexports_bzl",
     srcs = ["reexports.bzl"],
-    visibility = ["//python:__pkg__"],
+    visibility = [
+        "//docs:__pkg__",
+        "//python:__pkg__",
+    ],
     deps = [":bazel_tools_bzl"],
 )
 

python/py_binary.bzl

@@ -0,0 +1,19 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Public entry point for py_binary."""
+
+load("//python/private:reexports.bzl", _py_binary = "py_binary")
+
+py_binary = _py_binary

python/py_import.bzl

@@ -0,0 +1,67 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Public entry point for py_import rule."""
+
+load(":py_info.bzl", "PyInfo")
+
+def _py_import_impl(ctx):
+    # See https://github.com/bazelbuild/bazel/blob/0.24.0/src/main/java/com/google/devtools/build/lib/bazel/rules/python/BazelPythonSemantics.java#L104 .
+    import_paths = [
+        "/".join([ctx.workspace_name, x.short_path])
+        for x in ctx.files.srcs
+    ]
+
+    return [
+        DefaultInfo(
+            default_runfiles = ctx.runfiles(ctx.files.srcs, collect_default = True),
+        ),
+        PyInfo(
+            transitive_sources = depset(transitive = [
+                dep[PyInfo].transitive_sources
+                for dep in ctx.attr.deps
+            ]),
+            imports = depset(direct = import_paths, transitive = [
+                dep[PyInfo].imports
+                for dep in ctx.attr.deps
+            ]),
+        ),
+    ]
+
+py_import = rule(
+    doc = """This rule allows the use of Python packages as dependencies.
+
+    It imports the given `.egg` file(s), which might be checked in source files,
+    fetched externally as with `http_file`, or produced as outputs of other rules.
+
+    It may be used like a `py_library`, in the `deps` of other Python rules.
+
+    This is similar to [java_import](https://docs.bazel.build/versions/master/be/java.html#java_import).
+    """,
+    implementation = _py_import_impl,
+    attrs = {
+        "deps": attr.label_list(
+            doc = "The list of other libraries to be linked in to the " +
+                  "binary target.",
+            providers = [PyInfo],
+        ),
+        "srcs": attr.label_list(
+            doc = "The list of Python package files provided to Python targets " +
+                  "that depend on this target. Note that currently only the .egg " +
+                  "format is accepted. For .whl files, try the whl_library rule. " +
+                  "We accept contributions to extend py_import to handle .whl.",
+            allow_files = [".egg"],
+        ),
+    },
+)

python/py_info.bzl

@@ -0,0 +1,19 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Public entry point for PyInfo."""
+
+load("//python/private:reexports.bzl", "internal_PyInfo")
+
+PyInfo = internal_PyInfo