DOC-6979 Publish the versioned pages in sitemap.xml

.github/workflows/main.yml

@@ -218,13 +218,33 @@ jobs:
 
         hugo -d "output"
 
+        # Hugo writes one sitemap at the root of the build output, but deploy_kubernetes
+        # uploads only the versioned subdirectory, so that file never reaches the
+        # bucket -- and the latest build deletes these version directories before it
+        # builds, so its sitemap cannot list them either. Result: no sitemap anywhere
+        # listed a versioned page. Write a subtree sitemap into the directory that
+        # does ship, so the existing rsync picks it up with no deploy changes.
+        python3 build/generate_version_sitemap.py \
+          --sitemap output/sitemap.xml \
+          --subtree "operate/kubernetes/${version}" \
+          --output "output/operate/kubernetes/${version}/sitemap.xml"
+
     - name: Upload artifact
       uses: actions/upload-artifact@v4
       with:
         name: build-kubernetes-${{ matrix.version }}
         path: output/
         retention-days: 1
 
+    # Uploaded separately as well so deploy_version_sitemaps can collect every
+    # version's sitemap without downloading four dozen full site builds.
+    - name: Upload Kubernetes ${{ matrix.version }} sitemap
+      uses: actions/upload-artifact@v4
+      with:
+        name: sitemap-kubernetes-${{ matrix.version }}
+        path: output/operate/kubernetes/${{ matrix.version }}/sitemap.xml
+        retention-days: 1
+
   # Build RS versions in parallel
   build_rs:
     name: Build RS ${{ matrix.version }}
@@ -289,13 +309,27 @@ jobs:
 
         hugo -d "output"
 
+        # See the Kubernetes build: the root sitemap is never deployed, so write one
+        # into the versioned subdirectory that is.
+        python3 build/generate_version_sitemap.py \
+          --sitemap output/sitemap.xml \
+          --subtree "operate/rs/${version}" \
+          --output "output/operate/rs/${version}/sitemap.xml"
+
     - name: Upload artifact
       uses: actions/upload-artifact@v4
       with:
         name: build-rs-${{ matrix.version }}
         path: output/
         retention-days: 1
 
+    - name: Upload RS ${{ matrix.version }} sitemap
+      uses: actions/upload-artifact@v4
+      with:
+        name: sitemap-rs-${{ matrix.version }}
+        path: output/operate/rs/${{ matrix.version }}/sitemap.xml
+        retention-days: 1
+
   # Build RDI versions in parallel
   build_rdi:
     name: Build RDI ${{ matrix.version }}
@@ -360,13 +394,27 @@ jobs:
 
         hugo -d "output"
 
+        # See the Kubernetes build: the root sitemap is never deployed, so write one
+        # into the versioned subdirectory that is.
+        python3 build/generate_version_sitemap.py \
+          --sitemap output/sitemap.xml \
+          --subtree "integrate/redis-data-integration/${version}" \
+          --output "output/integrate/redis-data-integration/${version}/sitemap.xml"
+
     - name: Upload artifact
       uses: actions/upload-artifact@v4
       with:
         name: build-rdi-${{ matrix.version }}
         path: output/
         retention-days: 1
 
+    - name: Upload RDI ${{ matrix.version }} sitemap
+      uses: actions/upload-artifact@v4
+      with:
+        name: sitemap-rdi-${{ matrix.version }}
+        path: output/integrate/redis-data-integration/${{ matrix.version }}/sitemap.xml
+        retention-days: 1
+
   # Build RedisVL versions in parallel
   build_redisvl:
     name: Build RedisVL ${{ matrix.version }}
@@ -431,13 +479,27 @@ jobs:
 
         hugo -d "output"
 
+        # See the Kubernetes build: the root sitemap is never deployed, so write one
+        # into the versioned subdirectory that is.
+        python3 build/generate_version_sitemap.py \
+          --sitemap output/sitemap.xml \
+          --subtree "develop/ai/redisvl/${version}" \
+          --output "output/develop/ai/redisvl/${version}/sitemap.xml"
+
     - name: Upload artifact
       uses: actions/upload-artifact@v4
       with:
         name: build-redisvl-${{ matrix.version }}
         path: output/
         retention-days: 1
 
+    - name: Upload RedisVL ${{ matrix.version }} sitemap
+      uses: actions/upload-artifact@v4
+      with:
+        name: sitemap-redisvl-${{ matrix.version }}
+        path: output/develop/ai/redisvl/${{ matrix.version }}/sitemap.xml
+        retention-days: 1
+
   # Deploy latest build to GCS
   deploy_latest:
     name: Deploy latest
@@ -790,6 +852,118 @@ jobs:
         fi
 
   # Deploy custom 404 page (only for the production latest build)
+  # Republish /sitemap.xml with the versioned pages folded in.
+  #
+  # deploy_latest publishes the sitemap Hugo rendered for the latest build, which by
+  # construction cannot list a versioned page: that build deletes the version
+  # directories before Hugo runs. The versioned builds do render those pages, but
+  # only their versioned subdirectory is deployed, so their sitemaps never ship.
+  # This job runs after both and overwrites the published file with the union --
+  # ~2,700 latest URLs plus ~3,100 versioned ones -- so the address the SEO team
+  # already references gains the missing pages with no change on their side.
+  #
+  # One flat urlset rather than a sitemap index, deliberately: their file is itself
+  # a sitemap index, and the protocol does not allow one index to nest inside
+  # another. At ~5,800 URLs this is well inside the 50,000 URL / 50 MB ceiling.
+  deploy_complete_sitemap:
+    name: Deploy complete sitemap
+    needs:
+      - discover_versions
+      - build_kubernetes
+      - build_rs
+      - build_rdi
+      - build_redisvl
+      # deploy_latest mirrors public -> docs/<path> with `-d`, which would delete
+      # this file. Same gating reason as the versioned deploy jobs.
+      - deploy_latest
+    # A build job is skipped when its product has no version directories (RDI has
+    # none today), and a skipped `needs` would skip this job too -- hence
+    # `!cancelled()` rather than a plain dependency. Only deploy_latest is
+    # load-bearing; the rest contribute artifacts when they run. The last clause
+    # stands the job down entirely when nothing is versioned, so that an empty
+    # artifact set is a no-op instead of a failed merge.
+    if: >-
+      ${{ !cancelled()
+      && needs.deploy_latest.result == 'success'
+      && !(needs.discover_versions.outputs.kubernetes_versions == '[]'
+           && needs.discover_versions.outputs.rs_versions == '[]'
+           && needs.discover_versions.outputs.rdi_versions == '[]'
+           && needs.discover_versions.outputs.redisvl_versions == '[]') }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    env:
+      PROD_PROJECT_ID: ${{ secrets.GCP_PROJECT_PROD }}
+      PROD_SERVICE_ACCOUNT: ${{ secrets.PROD_SERVICE_ACCOUNT }}
+      PROD_WORKLOAD_IDENTITY_PROVIDER: ${{ secrets.PROD_WORKLOAD_IDENTITY_PROVIDER }}
+
+    steps:
+    - name: Check the branch out
+      uses: actions/checkout@v4
+
+    - name: Download the versioned sitemaps
+      uses: actions/download-artifact@v4
+      with:
+        pattern: sitemap-*
+        path: sitemaps/
+
+    # The latest build's own sitemap is the other half of the union. Taken from its
+    # artifact rather than from the bucket so the merge reflects the build being
+    # deployed rather than whatever a concurrent run happened to leave published.
+    - name: Download the latest build's sitemap
+      uses: actions/download-artifact@v4
+      with:
+        name: build-latest
+        path: latest-build/
+
+    - name: Merge every sitemap into one
+      run: |
+        mkdir -p sitemaps/latest
+        cp latest-build/sitemap.xml sitemaps/latest/sitemap.xml
+
+        # One sitemap per discovered version, plus the latest build's. Fewer means a
+        # matrix build failed or dropped its artifact, and overwriting the published
+        # file with a partial one would silently drop those pages from search again --
+        # the exact regression this job exists to prevent.
+        expected=$( (
+          echo '${{ needs.discover_versions.outputs.kubernetes_versions }}'
+          echo '${{ needs.discover_versions.outputs.rs_versions }}'
+          echo '${{ needs.discover_versions.outputs.rdi_versions }}'
+          echo '${{ needs.discover_versions.outputs.redisvl_versions }}'
+        ) | jq -s 'map(length) | add')
+        expected=$((expected + 1))
+        echo "Expecting ${expected} sitemaps (versions + latest)"
+
+        python3 build/merge_sitemaps.py sitemaps \
+          --output sitemap.xml \
+          --expect "$expected"
+
+    - name: 'Google auth'
+      uses: 'google-github-actions/auth@v2'
+      with:
+        project_id: '${{ env.PROD_PROJECT_ID }}'
+        service_account: '${{ env.PROD_SERVICE_ACCOUNT }}'
+        workload_identity_provider: '${{ env.PROD_WORKLOAD_IDENTITY_PROVIDER }}'
+
+    - name: 'Set up Cloud SDK'
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        project_id: '${{ env.PROD_PROJECT_ID }}'
+        version: '>= 363.0.0'
+
+    - name: Deploy the complete sitemap to GCS
+      run: |
+        bucket_path="${{ needs.discover_versions.outputs.bucket_path }}"
+
+        # Mirrors the versioned deploy jobs, which act on latest and staging only.
+        if [[ "$bucket_path" == "latest" || "$bucket_path" == staging/* ]]; then
+          # Both destinations deploy_latest mirrors to, so the two copies of
+          # sitemap.xml do not disagree about which pages exist.
+          gsutil cp sitemap.xml "gs://${BUCKET}/${bucket_path}/sitemap.xml"
+          gsutil cp sitemap.xml "gs://${BUCKET}/docs/${bucket_path}/sitemap.xml"
+        fi
+
   deploy_404:
     name: Deploy 404 page
     needs:

build/generate_version_sitemap.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Write a sitemap for one versioned subtree, inside the directory that ships.
+
+Every versioned matrix build in ``.github/workflows/main.yml`` renders a *whole*
+site: the version's content rsynced up to the unversioned content path, plus the
+rest of the docs. Hugo therefore writes one ``sitemap.xml`` at the root of the
+build output, listing all ~5,800 pages.
+
+But the deploy step for a versioned build uploads only the versioned
+subdirectory::
+
+    gsutil -m rsync -r -c -j html -d \\
+      "output/operate/rs/${version}" \\
+      "gs://${BUCKET}/docs/${bucket_path}/operate/rs/${version}"
+
+``output/sitemap.xml`` sits one or more levels *above* that directory, so it is
+never uploaded by any job. The ``latest`` build deletes the version directories
+before building, so its sitemap cannot list those pages either. The result is
+that no sitemap anywhere lists a versioned page -- 3,134 live URLs as of this
+writing, measured against a local unrestricted build.
+
+This script closes that gap without touching the deploy commands: it filters the
+rendered sitemap down to the pages under one versioned subtree and writes the
+result *into* the directory that already ships, so the existing rsync picks it up.
+``merge_sitemaps.py`` then folds every version's file, plus the latest build's, into
+the single published ``sitemap.xml``.
+
+The locs need no rewriting. Every page under a version directory carries explicit
+``url:`` frontmatter pinning its versioned path (all 435 files under
+``content/operate/rs/7.8`` do), so Hugo already computes the correct public
+permalink -- ``https://redis.io/docs/latest/operate/rs/7.8/...`` -- even though
+the content was rsynced to the unversioned path before the build. We copy whole
+``<url>`` elements, so ``lastmod`` comes through as Hugo computed it from git.
+
+Exits non-zero when the subtree matches nothing. That is the regression guard: if
+a future change stops those pages being emitted at their versioned URLs, this
+fails the build rather than silently shipping an empty sitemap.
+
+Run with ``pytest build/test_sitemaps.py`` for the tests.
+"""
+
+import argparse
+import logging
+import os
+import sys
+import xml.etree.ElementTree as ET
+from urllib.parse import urlparse
+
+SITEMAP_NS = "http://www.sitemaps.org/schemas/sitemap/0.9"
+XHTML_NS = "http://www.w3.org/1999/xhtml"
+
+logger = logging.getLogger("generate_version_sitemap")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--sitemap", default="output/sitemap.xml",
+                        help="sitemap Hugo rendered at the build root")
+    parser.add_argument("--subtree", required=True,
+                        help="versioned path to keep, e.g. operate/rs/7.8")
+    parser.add_argument("--output", required=True,
+                        help="where to write the filtered sitemap")
+    parser.add_argument("--allow-empty", action="store_true",
+                        help="warn instead of failing when nothing matches")
+    return parser.parse_args()
+
+
+def in_subtree(loc: str, subtree: str) -> bool:
+    """Is ``loc`` the subtree root or a page beneath it?
+
+    Compared on the URL *path* so that a baseURL which itself contains the
+    subtree string cannot widen the match, and segment-anchored so that
+    ``operate/rs/7.8`` does not swallow a future ``operate/rs/7.8-rc1``.
+    """
+    path = urlparse(loc).path
+    marker = "/" + subtree.strip("/")
+    return path.rstrip("/").endswith(marker) or (marker + "/") in path
+
+
+def filter_sitemap(xml_text: str, subtree: str) -> tuple[str, int]:
+    """Return a sitemap holding only the ``<url>`` entries under ``subtree``."""
+    ET.register_namespace("", SITEMAP_NS)
+    ET.register_namespace("xhtml", XHTML_NS)
+
+    source = ET.fromstring(xml_text)
+    kept = ET.Element(f"{{{SITEMAP_NS}}}urlset")
+
+    for url in source.findall(f"{{{SITEMAP_NS}}}url"):
+        loc = url.find(f"{{{SITEMAP_NS}}}loc")
+        if loc is not None and loc.text and in_subtree(loc.text, subtree):
+            kept.append(url)
+
+    ET.indent(kept, space="  ")
+    body = ET.tostring(kept, encoding="unicode")
+    header = '<?xml version="1.0" encoding="utf-8" standalone="yes"?>\n'
+    return header + body + "\n", len(kept)
+
+
+def main() -> int:
+    logging.basicConfig(level=logging.INFO, format="%(name)s: %(message)s")
+    args = parse_args()
+
+    if not os.path.isfile(args.sitemap):
+        logger.error("no sitemap at %s -- did hugo run?", args.sitemap)
+        return 1
+
+    with open(args.sitemap, encoding="utf-8") as handle:
+        xml_text = handle.read()
+
+    document, count = filter_sitemap(xml_text, args.subtree)
+
+    if not count:
+        message = "%s matched no URLs in %s"
+        if not args.allow_empty:
+            logger.error(message, args.subtree, args.sitemap)
+            logger.error("versioned pages are not being emitted at their "
+                         "versioned URLs -- check the url: frontmatter")
+            return 1
+        logger.warning(message, args.subtree, args.sitemap)
+
+    os.makedirs(os.path.dirname(args.output) or ".", exist_ok=True)
+    with open(args.output, "w", encoding="utf-8") as handle:
+        handle.write(document)
+
+    logger.info("wrote %d URLs for %s to %s", count, args.subtree, args.output)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())