sphinxdocs: add docs; support sources from other directories (bazel-contrib#2128)

Documents how to use Sphinx syntax when writing docs. There's a variety
of features the `sphinx_bzl` plugin enables, but without docs, they're
somewhat hard to discover and figure out how to use.

Because sphinxdocs is almost entirely separate, adding its docs under
`//sphinxdocs/docs` is a more natural fit. Unfortunately, this became
very verbose, repetitive, and tedious, for two reasons:
1. The only way `sphinx_docs` could accept files from other directories
was using the `rename_srcs` arg and manually renaming files one-by-one.
2. Similarly, `sphinx_stardocs` required a one-by-one mapping of each
bzl file to its output file, which then had to be repeated in
`rename_srcs`.

To fix (1), the `sphinx_docs.deps` attribute and `sphinx_docs_library`
rule are added. The library targets collect files, and `sphinx_docs`
moves then into the final Sphinx sources directory.

To fix (2), the `sphinx_stardoc.srcs` attribute is added, which accepts
`bzl_library` targets. I noticed that, in almost all cases, the output
name was simply the input name with the `.md` extension, so the rule now
does that by default. For special cases, the `sphinx_stardoc` (singular)
rule can be called directly.

Also:
* Adds `bzl:rule` as a cross reference lookup role
* Removes some defunct stuff relating to the stardoc template files that
aren't used anymore.
* Disables warnings from the autosectionlabel extension. These were
spamming warnings because CHANGELOG.md has many headers with the same
name.
* Adds more entries to bazel inventory (all of native and ctx)

Author: rickeylev

docs/sphinx/BUILD.bazel

@@ -19,7 +19,7 @@ load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")  # buildifier: dis
 load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER")  # buildifier: disable=bzl-visibility
 load("//sphinxdocs:readthedocs.bzl", "readthedocs_install")
 load("//sphinxdocs:sphinx.bzl", "sphinx_build_binary", "sphinx_docs")
-load("//sphinxdocs:sphinx_stardoc.bzl", "sphinx_stardocs")
+load("//sphinxdocs:sphinx_stardoc.bzl", "sphinx_stardoc", "sphinx_stardocs")
 
 # We only build for Linux and Mac because:
 # 1. The actual doc process only runs on Linux
@@ -38,9 +38,7 @@ _TARGET_COMPATIBLE_WITH = select({
 # * `ibazel build //docs/sphinx:docs` to automatically rebuild docs
 sphinx_docs(
     name = "docs",
-    srcs = [
-        ":bzl_api_docs",
-    ] + glob(
+    srcs = glob(
         include = [
             "*.md",
             "**/*.md",
@@ -66,43 +64,56 @@ sphinx_docs(
     strip_prefix = package_name() + "/",
     tags = ["docs"],
     target_compatible_with = _TARGET_COMPATIBLE_WITH,
+    deps = [
+        ":bzl_api_docs",
+        "//sphinxdocs/docs:docs_lib",
+    ],
 )
 
 sphinx_stardocs(
     name = "bzl_api_docs",
-    docs = {
-        "api/python/cc/py_cc_toolchain.md": dict(
-            dep = "//python/private:py_cc_toolchain_bzl",
-            input = "//python/private:py_cc_toolchain_rule.bzl",
-            public_load_path = "//python/cc:py_cc_toolchain.bzl",
-        ),
-        "api/python/cc/py_cc_toolchain_info.md": "//python/cc:py_cc_toolchain_info_bzl",
-        "api/python/defs.md": "//python:defs_bzl",
-        "api/python/entry_points/py_console_script_binary.md": "//python/entry_points:py_console_script_binary_bzl",
-        "api/python/packaging.md": "//python:packaging_bzl",
-        "api/python/pip.md": "//python:pip_bzl",
-        "api/python/private/common/py_binary_rule_bazel.md": "//python/private/common:py_binary_rule_bazel_bzl",
-        "api/python/private/common/py_library_rule_bazel.md": "//python/private/common:py_library_rule_bazel_bzl",
-        "api/python/private/common/py_runtime_rule.md": "//python/private/common:py_runtime_rule_bzl",
-        "api/python/private/common/py_test_rule_bazel.md": "//python/private/common:py_test_rule_bazel_bzl",
-        "api/python/py_binary.md": "//python:py_binary_bzl",
-        "api/python/py_cc_link_params_info.md": "//python:py_cc_link_params_info_bzl",
-        "api/python/py_library.md": "//python:py_library_bzl",
-        "api/python/py_runtime.md": "//python:py_runtime_bzl",
-        "api/python/py_runtime_info.md": "//python:py_runtime_info_bzl",
-        "api/python/py_runtime_pair.md": dict(
-            dep = "//python/private:py_runtime_pair_rule_bzl",
-            input = "//python/private:py_runtime_pair_rule.bzl",
-            public_load_path = "//python:py_runtime_pair.bzl",
-        ),
-        "api/python/py_test.md": "//python:py_test_bzl",
-    } | ({
+    srcs = [
+        "//python:defs_bzl",
+        "//python:packaging_bzl",
+        "//python:pip_bzl",
+        "//python:py_binary_bzl",
+        "//python:py_cc_link_params_info_bzl",
+        "//python:py_library_bzl",
+        "//python:py_runtime_bzl",
+        "//python:py_runtime_info_bzl",
+        "//python:py_test_bzl",
+        "//python/cc:py_cc_toolchain_info_bzl",
+        "//python/entry_points:py_console_script_binary_bzl",
+        "//python/private/common:py_binary_rule_bazel_bzl",
+        "//python/private/common:py_library_rule_bazel_bzl",
+        "//python/private/common:py_runtime_rule_bzl",
+        "//python/private/common:py_test_rule_bazel_bzl",
+    ] + ([
         # Bazel 6 + Stardoc isn't able to parse something about the python bzlmod extension
-        "api/python/extensions/python.md": "//python/extensions:python_bzl",
-    } if IS_BAZEL_7_OR_HIGHER else {}) | ({
+        "//python/extensions:python_bzl",
+    ] if IS_BAZEL_7_OR_HIGHER else []) + ([
         # This depends on @pythons_hub, which is only created under bzlmod,
-        "api/python/extensions/pip.md": "//python/extensions:pip_bzl",
-    } if IS_BAZEL_7_OR_HIGHER and BZLMOD_ENABLED else {}),
+        "//python/extensions:pip_bzl",
+    ] if IS_BAZEL_7_OR_HIGHER and BZLMOD_ENABLED else []),
+    prefix = "api/",
+    tags = ["docs"],
+    target_compatible_with = _TARGET_COMPATIBLE_WITH,
+)
+
+sphinx_stardoc(
+    name = "py_cc_toolchain",
+    src = "//python/private:py_cc_toolchain_rule.bzl",
+    prefix = "api/",
+    public_load_path = "//python/cc:py_cc_toolchain.bzl",
+    tags = ["docs"],
+    target_compatible_with = _TARGET_COMPATIBLE_WITH,
+    deps = ["//python/cc:py_cc_toolchain_bzl"],
+)
+
+sphinx_stardoc(
+    name = "py_runtime_pair",
+    src = "//python/private:py_runtime_pair_rule_bzl",
+    public_load_path = "//python:py_runtime_pair.bzl",
     tags = ["docs"],
     target_compatible_with = _TARGET_COMPATIBLE_WITH,
 )

docs/sphinx/_stardoc_footer.md

@@ -144,7 +144,12 @@
 # -- Options for EPUB output
 epub_show_urls = "footnote"
 
-suppress_warnings = []
+suppress_warnings = [
+    # The autosectionlabel extension turns header titles into referencable
+    # names. Unfortunately, CHANGELOG.md has many duplicate header titles,
+    # which creates lots of warning spam. Just ignore them.
+    "autosectionlabel.*"
+]
 
 
 def setup(app):

docs/sphinx/conf.py

@@ -67,6 +67,7 @@ support
 Changelog <changelog>
 api/index
 environment-variables
+Sphinxdocs <sphinxdocs/index>
 glossary
 genindex
 ```

docs/sphinx/index.md

@@ -47,6 +47,12 @@ bzl_library(
     deps = ["//sphinxdocs/private:sphinx_bzl"],
 )
 
+bzl_library(
+    name = "sphinx_docs_library_bzl",
+    srcs = ["sphinx_docs_library.bzl"],
+    deps = ["//sphinxdocs/private:sphinx_docs_library_macro_bzl"],
+)
+
 bzl_library(
     name = "sphinx_stardoc_bzl",
     srcs = ["sphinx_stardoc.bzl"],

sphinxdocs/BUILD.bazel

@@ -0,0 +1,55 @@
+load("//python/private:util.bzl", "IS_BAZEL_7_OR_HIGHER")  # buildifier: disable=bzl-visibility
+load("//sphinxdocs:sphinx_docs_library.bzl", "sphinx_docs_library")
+load("//sphinxdocs:sphinx_stardoc.bzl", "sphinx_stardocs")
+
+package(default_visibility = ["//:__subpackages__"])
+
+# We only build for Linux and Mac because:
+# 1. The actual doc process only runs on Linux
+# 2. Mac is a common development platform, and is close enough to Linux
+#    it's feasible to make work.
+# Making CI happy under Windows is too much of a headache, though, so we don't
+# bother with that.
+_TARGET_COMPATIBLE_WITH = select({
+    "@platforms//os:linux": [],
+    "@platforms//os:macos": [],
+    "//conditions:default": ["@platforms//:incompatible"],
+}) if IS_BAZEL_7_OR_HIGHER else ["@platforms//:incompatible"]
+
+sphinx_docs_library(
+    name = "docs_lib",
+    deps = [
+        ":artisian_api_docs",
+        ":artisian_docs",
+        ":bzl_docs",
+    ],
+)
+
+sphinx_docs_library(
+    name = "artisian_docs",
+    srcs = glob(
+        ["**/*.md"],
+        exclude = ["api/**"],
+    ),
+    prefix = "sphinxdocs/",
+)
+
+sphinx_docs_library(
+    name = "artisian_api_docs",
+    srcs = glob(
+        ["api/**/*.md"],
+    ),
+)
+
+sphinx_stardocs(
+    name = "bzl_docs",
+    srcs = [
+        "//sphinxdocs:readthedocs_bzl",
+        "//sphinxdocs:sphinx_bzl",
+        "//sphinxdocs:sphinx_docs_library_bzl",
+        "//sphinxdocs:sphinx_stardoc_bzl",
+        "//sphinxdocs/private:sphinx_docs_library_bzl",
+    ],
+    prefix = "api/",
+    target_compatible_with = _TARGET_COMPATIBLE_WITH,
+)

sphinxdocs/docs/BUILD.bazel

@@ -0,0 +1,29 @@
+:::{bzl:currentfile} //sphinxdocs:BUILD.bazel
+:::
+
+# //sphinxdocs
+
+:::{bzl:flag} extra_defines
+Additional `-D` values to add to every Sphinx build.
+
+This is a list flag. Multiple uses are accumulated.
+
+This is most useful for overriding e.g. the version when performing
+release builds.
+:::
+
+:::{bzl:flag} extra_env
+Additional environment variables to for every Sphinx build.
+
+This is a list flag. Multiple uses are accumulated. Values are `key=value`
+format.
+:::
+
+:::{bzl:flag} quiet
+Whether to add the `-q` arg to Sphinx invocations.
+
+This is a boolean flag.
+
+This is useful for debugging invocations or developing extensions. The Sphinx
+`-q` flag causes sphinx to produce additional output on stdout.
+:::

sphinxdocs/docs/api/sphinxdocs/index.md

@@ -0,0 +1,11 @@
+:::{bzl:currentfile} //sphinxdocs/inventories:BUILD.bazel
+:::
+
+# //sphinxdocs/inventories
+
+:::{bzl:target} bazel_inventory
+A Sphinx inventory of Bazel objects.
+
+By including this target in your Sphinx build and enabling intersphinx, cross
+references to builtin Bazel objects can be written.
+:::

sphinxdocs/docs/api/sphinxdocs/inventories/index.md

@@ -0,0 +1,20 @@
+# Docgen using Sphinx with Bazel
+
+The `sphinxdocs` project allows using Bazel to run Sphinx to generate
+documentation. It comes with:
+
+* Rules for running Sphinx
+* Rules for generating documentation for Starlark code.
+* A Sphinx plugin for documenting Starlark and Bazel objects.
+* Rules for readthedocs build integration.
+
+While it is primarily oriented towards docgen for Starlark code, the core of it
+is agnostic as to what is being documented.
+
+
+```{toctree}
+:hidden:
+
+starlark-docgen
+sphinx-bzl
+```