fix(ci) harden offline sigstore bundle verification

.github/workflows/docker-riverproui.yaml

@@ -37,7 +37,7 @@ jobs:
     name: "Build image: riverproui"
     runs-on: ubuntu-latest
     # Skip this job when manually dispatched with verify_only=true
-    if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.verify_only == 'true') }}
+    if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.verify_only) }}
     env:
       ECR_ACCOUNT_ID: ${{ secrets.ECR_CACHE_AWS_ACCOUNT_ID }}
       ECR_ROLE_ARN: ${{ secrets.ECR_CACHE_ROLE_ARN }}
@@ -124,7 +124,7 @@ jobs:
     needs:
       - build-riverproui
     # Skip this job when manually dispatched with verify_only=true
-    if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.verify_only == 'true') }}
+    if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.verify_only) }}
     permissions:
       contents: read
       id-token: write
@@ -363,7 +363,7 @@ jobs:
     # Only run on release tag events (refs/tags/riverproui/vX.Y.Z),
     # or when manually forced via workflow_dispatch: force_prefetch=true.
     # For workflow_dispatch, inputs.ref should be in the form 'riverproui/vX.Y.Z'.
-    if: startsWith(github.ref, 'refs/tags/riverproui/v') || (github.event_name == 'workflow_dispatch' && (startsWith(inputs.ref, 'riverproui/v') || inputs.force_prefetch == 'true'))
+    if: startsWith(github.ref, 'refs/tags/riverproui/v') || (github.event_name == 'workflow_dispatch' && (startsWith(inputs.ref, 'riverproui/v') || inputs.force_prefetch))
     permissions:
       contents: read
     steps:
@@ -475,7 +475,7 @@ jobs:
           FORCE_FETCH_SECRET: ${{ secrets.FORCE_FETCH_SECRET }}
           REGISTRY_MANIFEST_URL: https://riverqueue.com/v2/riverproui/manifests
         run: |
-          bash scripts/prefetch-buildx-attestation-manifests.sh /tmp/index-manifest-docker.json
+          bash scripts/ci/prefetch-buildx-attestation-manifests.sh /tmp/index-manifest-docker.json
 
       - name: Fetch amd64 manifest with crane (Docker media type)
         run: crane manifest "$IMAGE_NAME@${{ steps.platform-digests.outputs.amd64_digest }}" > /tmp/amd64-manifest-docker.json
@@ -506,6 +506,12 @@ jobs:
       # If neither MANIFEST_DIGEST (from merge job) nor ecr_manifest_digest (manual input)
       # is available, we skip index referrer prefetch rather than fail.
       - name: Prefetch all referrers content
+        env:
+          AUTH_USER: river
+          AUTH_PASSWORD: ${{ secrets.RIVERPRO_GO_MOD_CREDENTIAL }}
+          FORCE_FETCH_SECRET: ${{ secrets.FORCE_FETCH_SECRET }}
+          REGISTRY_MANIFEST_URL: https://riverqueue.com/v2/riverproui/manifests
+          REGISTRY_REFERRERS_URL: https://riverqueue.com/v2/riverproui/referrers
         run: |
           # Include both the ECR-derived index digest (attestations live here)
           # and the live registry's index digest (may differ but could gain referrers).
@@ -518,24 +524,7 @@ jobs:
             echo "Note: no index subject digest provided; skipping index referrers prefetch."
           fi
           subjects+=("${{ steps.resolve-digest.outputs.digest }}" "${{ steps.platform-digests.outputs.amd64_digest }}" "${{ steps.platform-digests.outputs.arm64_digest }}")
-          for subject in "${subjects[@]}"; do
-            oras discover --format json "$IMAGE_NAME@$subject" | tee /tmp/referrers.json
-            digests=$(jq -r '.manifests[]?.digest // empty' /tmp/referrers.json)
-            for d in $digests; do
-              tmpfile=$(mktemp)
-              # Fetch the full referrer manifest to a file so we can parse layers + config
-              oras manifest fetch --output "$tmpfile" "$IMAGE_NAME@$d"
-              # Prefetch config blob (often empty but warms cache)
-              cfg=$(jq -r '.config.digest // empty' "$tmpfile")
-              if [ -n "$cfg" ]; then
-                oras blob fetch --output /dev/null "$IMAGE_NAME@$cfg"
-              fi
-              # Prefetch any layer blobs (DSSE envelope, bundle, etc.)
-              jq -r '.layers[]?.digest // empty' "$tmpfile" | while read -r ld; do
-                [ -n "$ld" ] && oras blob fetch --output /dev/null "$IMAGE_NAME@$ld"
-              done
-            done
-          done
+          bash scripts/ci/prefetch-referrers-content.sh "${subjects[@]}"
 
       # Verify index (if digest is available) and per-arch attestations against live registry.
       # Index-level attestation is bound to the ECR index digest; provide it via MANIFEST_DIGEST
@@ -576,55 +565,23 @@ jobs:
 
       # Offline verification of Sigstore bundle: only attempt for index if digest provided.
       - name: Verify Sigstore bundle against subject bytes (offline crypto)
+        env:
+          AUTH_USER: river
+          AUTH_PASSWORD: ${{ secrets.RIVERPRO_GO_MOD_CREDENTIAL }}
+          FORCE_FETCH_SECRET: ${{ secrets.FORCE_FETCH_SECRET }}
+          REGISTRY_MANIFEST_URL: https://riverqueue.com/v2/riverproui/manifests
+          REGISTRY_REFERRERS_URL: https://riverqueue.com/v2/riverproui/referrers
         run: |
           FALLBACK="${{ inputs.ecr_manifest_digest }}"
           INDEX_SUBJECT="${MANIFEST_DIGEST:-$FALLBACK}"
-          verify_bundle () {
-            subject="$1"
-            # Exact subject bytes
-            oras manifest fetch --output /tmp/subject.json "$IMAGE_NAME@$subject"
-
-            # Find *all* bundle candidates for this subject (both media-type spellings)
-            refjson=$(mktemp)
-            oras discover --format json "$IMAGE_NAME@$subject" > "$refjson"
-            mapfile -t bundle_mfs < <(jq -r '.manifests[]
-              | select(.artifactType=="application/vnd.dev.sigstore.bundle.v0.3+json"
-                       or .artifactType=="application/vnd.dev.sigstore.bundle+json;version=0.3")
-              | .digest' "$refjson")
-
-            ok=0
-            for bmf in "${bundle_mfs[@]}"; do
-              [ -z "$bmf" ] && continue
-              mf=$(mktemp)
-              oras manifest fetch --output "$mf" "$IMAGE_NAME@$bmf"
-              bundle_blob=$(jq -r '.layers[0].digest' "$mf")
-              oras blob fetch --output /tmp/bundle.json "$IMAGE_NAME@$bundle_blob"
-
-              if cosign verify-blob-attestation \
-                    --bundle /tmp/bundle.json \
-                    --new-bundle-format \
-                    --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \
-                    --certificate-identity-regexp '^https://github.com/riverqueue/riverui/.*' \
-                    /tmp/subject.json > /tmp/verify.out 2>&1; then
-                echo "Verified OK with bundle $bmf"
-                ok=1
-                break
-              fi
-            done
-
-            if [ "$ok" -ne 1 ]; then
-              echo "No matching bundle verified for subject $subject"
-              cat /tmp/verify.out || true
-              exit 1
-            fi
-          }
           if [ -n "$INDEX_SUBJECT" ]; then
-            verify_bundle "$INDEX_SUBJECT"
+            bash scripts/ci/verify-sigstore-bundles-offline.sh "$INDEX_SUBJECT"
           else
             echo "Skipping offline index bundle verification: no MANIFEST_DIGEST or ecr_manifest_digest provided."
           fi
-          verify_bundle "${{ steps.platform-digests.outputs.amd64_digest }}"
-          verify_bundle "${{ steps.platform-digests.outputs.arm64_digest }}"
+          bash scripts/ci/verify-sigstore-bundles-offline.sh \
+            "${{ steps.platform-digests.outputs.amd64_digest }}" \
+            "${{ steps.platform-digests.outputs.arm64_digest }}"
 
       - name: Upload debug artifacts
         if: failure() || cancelled()

docs/development.md

@@ -89,13 +89,47 @@ $ npm run build
 
 2. Prepare a PR with the changes, updating `CHANGELOG.md` with any necessary additions at the same time. Have it reviewed and merged.
 
-3. Upon merge, pull down the changes, tag each module with the new version, and push the new tags:
+3. Upon merge, pull down the changes, tag the main riverui module with the new version, and push the new tag:
 
    ```shell
    git pull origin master
    git tag $VERSION
-   git tag riverproui/$VERSION -m "release riverproui/$VERSION"
    git push --tags
    ```
 
 4. The build will cut a new release and create binaries automatically, but it won't have a good release message. Go the [release list](https://github.com/riverqueue/riverui/releases), find `$VERSION` and change the description to the release content in `CHANGELOG.md` (again, the build will have to finish first).
+
+### Releasing riverproui
+
+The `riverproui` submodule depends on the top level `riverui` module and in development it is customary to leave a `replace` directive in its `go.mod` so that it can be developed against the live local version. However, this `replace` directive makes it incompatible with `go install ...@latest`.
+
+As such, we must use a two-phase release for these modules:
+
+1. Release `riverui` with an initial version (i.e. all the steps above).
+
+2. Comment out `replace` directives to riverui `./riverproui/go.mod`. These were probably needed for developing the new feature, but need to be removed because they prevent the module from being `go install`-able.
+
+3. From `./riverproui`, `go get` to upgrade to the main package versions were just released (make sure you're getting `$VERSION` and not thwarted by shenanigans in Go's module proxy):
+
+   ```shell
+   cd ./riverproui
+   go get -u riverqueue.com/riverui@$VERSION
+   ```
+
+4. Run `go mod tidy`:
+
+   ```shell
+   go mod tidy
+   ```
+
+5. Prepare a PR with the changes. Have it reviewed and merged.
+
+6. Pull the changes back down, add a tag for `riverproui/$VERSION`, and push it to GitHub:
+
+   ```shell
+   git pull origin master
+   git tag riverproui/$VERSION -m "release riverproui/$VERSION"
+   git push --tags
+   ```
+
+   The main `$VERSION` tag and `riverproui/$VERSION` will point to different commits, and although a little odd, is tolerable.

scripts/ci/prefetch-referrers-content.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Prefetch referrers for one or more OCI subjects (digests), and warm their blobs.
+#
+# Required env:
+#   - IMAGE_NAME: e.g. "riverqueue.com/riverproui"
+#   - AUTH_USER: basic auth username
+#   - AUTH_PASSWORD: basic auth password
+#   - FORCE_FETCH_SECRET: value for X-Force-Fetch-From-Upstream (best-effort)
+#   - REGISTRY_MANIFEST_URL: e.g. "https://riverqueue.com/v2/riverproui/manifests"
+#   - REGISTRY_REFERRERS_URL: e.g. "https://riverqueue.com/v2/riverproui/referrers"
+#
+# Usage:
+#   bash scripts/ci/prefetch-referrers-content.sh sha256:... sha256:...
+
+: "${IMAGE_NAME:?IMAGE_NAME is required}"
+: "${AUTH_USER:?AUTH_USER is required}"
+: "${AUTH_PASSWORD:?AUTH_PASSWORD is required}"
+: "${FORCE_FETCH_SECRET:?FORCE_FETCH_SECRET is required}"
+: "${REGISTRY_MANIFEST_URL:?REGISTRY_MANIFEST_URL is required}"
+: "${REGISTRY_REFERRERS_URL:?REGISTRY_REFERRERS_URL is required}"
+
+ACCEPT_MANIFESTS="application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json"
+
+force_fetch_manifest_and_referrers() {
+  subject="$1"
+
+  # Best-effort: warm the pull-through cache for the subject bytes and its referrers index.
+  curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" \
+    -H "X-Force-Fetch-From-Upstream: ${FORCE_FETCH_SECRET}" \
+    -H "Accept: ${ACCEPT_MANIFESTS}" \
+    "${REGISTRY_MANIFEST_URL}/${subject}" -o /dev/null || true
+
+  curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" \
+    -H "X-Force-Fetch-From-Upstream: ${FORCE_FETCH_SECRET}" \
+    -H "Accept: application/vnd.oci.image.index.v1+json" \
+    "${REGISTRY_REFERRERS_URL}/${subject}" -o /dev/null || true
+}
+
+if [ "$#" -lt 1 ]; then
+  echo "usage: $0 <subject-digest> [<subject-digest> ...]" >&2
+  exit 2
+fi
+
+for subject in "$@"; do
+  [ -z "$subject" ] && continue
+
+  force_fetch_manifest_and_referrers "$subject"
+
+  # Discover and warm each referrer manifest and its blobs.
+  oras discover --format json "${IMAGE_NAME}@${subject}" | tee /tmp/referrers.json
+  # ORAS output shape differs by version/flags:
+  # - some versions return { "manifests": [...] }
+  # - some return { "referrers": [...] }
+  digests="$(jq -r '((.manifests // .referrers // []) | .[]? | .digest // empty)' /tmp/referrers.json)"
+  for d in $digests; do
+    [ -z "$d" ] && continue
+
+    tmpfile="$(mktemp)"
+    oras manifest fetch --output "$tmpfile" "${IMAGE_NAME}@${d}"
+
+    cfg="$(jq -r '.config.digest // empty' "$tmpfile")"
+    if [ -n "$cfg" ]; then
+      oras blob fetch --output /dev/null "${IMAGE_NAME}@${cfg}"
+    fi
+
+    jq -r '.layers[]?.digest // empty' "$tmpfile" | while read -r ld; do
+      [ -n "$ld" ] && oras blob fetch --output /dev/null "${IMAGE_NAME}@${ld}"
+    done
+  done
+done
+
+

scripts/ci/verify-sigstore-bundles-offline.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Offline verification of Sigstore bundles against exact subject bytes.
+#
+# Required env:
+#   - IMAGE_NAME: e.g. "riverqueue.com/riverproui"
+#   - AUTH_USER: basic auth username
+#   - AUTH_PASSWORD: basic auth password
+#   - FORCE_FETCH_SECRET: value for X-Force-Fetch-From-Upstream (best-effort)
+#   - REGISTRY_MANIFEST_URL: e.g. "https://riverqueue.com/v2/riverproui/manifests"
+#   - REGISTRY_REFERRERS_URL: e.g. "https://riverqueue.com/v2/riverproui/referrers"
+#
+# Usage:
+#   bash scripts/ci/verify-sigstore-bundles-offline.sh sha256:... sha256:... sha256:...
+#
+# Notes:
+# - We "force fetch" the subject + referrers index first to avoid ORAS limitations around custom headers.
+# - We retry discovery briefly to avoid eventual-consistency/race flakes in pull-through caches.
+
+: "${IMAGE_NAME:?IMAGE_NAME is required}"
+: "${AUTH_USER:?AUTH_USER is required}"
+: "${AUTH_PASSWORD:?AUTH_PASSWORD is required}"
+: "${FORCE_FETCH_SECRET:?FORCE_FETCH_SECRET is required}"
+: "${REGISTRY_MANIFEST_URL:?REGISTRY_MANIFEST_URL is required}"
+: "${REGISTRY_REFERRERS_URL:?REGISTRY_REFERRERS_URL is required}"
+
+ACCEPT_MANIFESTS="application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json"
+
+echo "oras: $(oras version 2>/dev/null || echo unknown)"
+echo "cosign: $(cosign version 2>/dev/null || echo unknown)"
+echo "jq: $(jq --version 2>/dev/null || echo unknown)"
+
+force_fetch_manifest_and_referrers() {
+  subject="$1"
+
+  curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" \
+    -H "X-Force-Fetch-From-Upstream: ${FORCE_FETCH_SECRET}" \
+    -H "Accept: ${ACCEPT_MANIFESTS}" \
+    "${REGISTRY_MANIFEST_URL}/${subject}" -o /dev/null || true
+
+  curl -fsS -u "${AUTH_USER}:${AUTH_PASSWORD}" \
+    -H "X-Force-Fetch-From-Upstream: ${FORCE_FETCH_SECRET}" \
+    -H "Accept: application/vnd.oci.image.index.v1+json" \
+    "${REGISTRY_REFERRERS_URL}/${subject}" -o /dev/null || true
+}
+
+verify_bundle_for_subject() {
+  subject="$1"
+  : > /tmp/verify.out
+
+  # Exact subject bytes (do not go through tag indirections).
+  oras manifest fetch --output /tmp/subject.json "${IMAGE_NAME}@${subject}"
+
+  bundle_mfs=()
+  refjson=""
+  for attempt in 1 2 3 4 5; do
+    refjson="$(mktemp)"
+    force_fetch_manifest_and_referrers "$subject"
+    oras discover --format json "${IMAGE_NAME}@${subject}" > "$refjson"
+
+    mapfile -t bundle_mfs < <(jq -r '((.manifests // .referrers // []) | .[]?)
+      | select(.artifactType=="application/vnd.dev.sigstore.bundle.v0.3+json"
+               or .artifactType=="application/vnd.dev.sigstore.bundle+json;version=0.3")
+      | .digest' "$refjson")
+
+    if [ ${#bundle_mfs[@]} -gt 0 ]; then
+      break
+    fi
+
+    if [ "$attempt" -lt 5 ]; then
+      echo "No bundle referrers discovered yet for ${subject} (attempt ${attempt}/5); retrying soon..."
+      sleep $((attempt * 2))
+    fi
+  done
+
+  ok=0
+  for bmf in "${bundle_mfs[@]}"; do
+    [ -z "$bmf" ] && continue
+
+    mf="$(mktemp)"
+    oras manifest fetch --output "$mf" "${IMAGE_NAME}@${bmf}"
+    bundle_blob="$(jq -r '.layers[0].digest' "$mf")"
+    oras blob fetch --output /tmp/bundle.json "${IMAGE_NAME}@${bundle_blob}"
+
+    if cosign verify-blob-attestation \
+      --bundle /tmp/bundle.json \
+      --new-bundle-format \
+      --certificate-oidc-issuer "https://token.actions.githubusercontent.com" \
+      --certificate-identity-regexp '^https://github.com/riverqueue/riverui/.*' \
+      /tmp/subject.json > /tmp/verify.out 2>&1; then
+      echo "Verified OK with bundle ${bmf} for subject ${subject}"
+      ok=1
+      break
+    fi
+  done
+
+  if [ "$ok" -ne 1 ]; then
+    echo "No matching bundle verified for subject ${subject}"
+    if [ ${#bundle_mfs[@]} -eq 0 ]; then
+      echo "Note: no bundle referrers were discovered for this subject. Referrers index JSON:"
+      cat "$refjson" || true
+    fi
+    cat /tmp/verify.out || true
+    exit 1
+  fi
+}
+
+if [ "$#" -lt 1 ]; then
+  echo "usage: $0 <subject-digest> [<subject-digest> ...]" >&2
+  exit 2
+fi
+
+for subject in "$@"; do
+  [ -z "$subject" ] && continue
+  verify_bundle_for_subject "$subject"
+done
+
+