ci: split docs deployment from publisher and deploy directly to pages (…

…nim-works#1370)

## Summary
Documentation will now be built using a separated workflow that will be
run whenever publisher finishes. This allows us to use the latest page
deployment code regardless of the commit that triggered publisher.

## Details
* Added  `get`  sub-command to  `release_manifest` , which allows one to
extract binary archive name from a release manifest.
* "Generated docs" artifacts are now removed. Instead the generated
documentation will be obtained from the release tarball for Linux amd64.
This allows docs publishing to run regardless of whether the artifact
for the latest release has expired or not.
* Documentation deployment will now skip  `gh-pages`  branch and deploy
directly using  `actions/deploy-pages` . This allows us to drop the
heavyweight  `gh-pages`  branch and remove the latency between push and
page being deployed.
* Documentation deployment can now be triggered on-demand in case of a
prior failure.
* Documentation deployment will now always be done using latest code in 
`devel` .
* Development documentation is also pushed to  `/devel`  sub-folder.
This is preliminary setup for versioned docs in the future.

---------

Co-authored-by: Saem Ghani <[email protected]>

.github/workflows/ci.yml

@@ -549,15 +549,6 @@ jobs:
           echo "archive=$archive" >> $GITHUB_OUTPUT
           echo "metadata=$metadata" >> $GITHUB_OUTPUT
 
-      - name: Upload docs to artifacts
-        if: matrix.target.shared_builder
-        uses: actions/upload-artifact@v4
-        with:
-          # If this name is updated, tweak publisher.yml
-          name: Generated docs
-          path: doc/html/
-          if-no-files-found: error
-
       - name: Upload release package to artifacts
         uses: actions/upload-artifact@v4
         with:

.github/workflows/deploy-docs.yml

@@ -0,0 +1,86 @@
+name: Deploy documentation
+on:
+  # Automatically run after any completed publish
+  workflow_run:
+    workflows:
+      - Publish built artifacts
+    types:
+      - completed
+
+  # For manual triggers
+  workflow_dispatch:
+
+# Run every script actions in bash
+defaults:
+  run:
+    shell: bash
+
+concurrency: doc-publisher
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    permissions:
+      id-token: write
+      pages: write
+
+    environment:
+      name: github-pages
+      url: ${{ steps.deploy.outputs.page_url }}
+
+    env:
+      # Triplet to obtain docs from
+      DOC_TARGET: x86_64-linux-gnu
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup latest compiler
+        uses: nim-works/[email protected]
+        with:
+          nimskull-version: "*" # Grab the latest nimskull-version
+
+      - name: Compile release_manifest
+        run: nim c -d:release -o:release_manifest tools/release_manifest.nim
+
+      - id: versions
+        name: Grab latest release version
+        run: |
+          # Stolen from asdf-nimskull
+          sort_versions() {
+            sed 'h; s/[+-]/./g; s/$/.z/; G; s/\n/ /' |
+              LC_ALL=C sort -t. -k 1,1n -k 2,2n -k 3,3n -k 4,4 -k 5,5n | awk '{print $2}'
+          }
+
+          all_tags=$(gh release list --json tagName --jq '.[] | .tagName')
+          latest=$(sort_versions <<<"$all_tags" | tail -n 1)
+
+          echo "Latest devel is: $latest"
+          echo "devel=$latest" >> "$GITHUB_OUTPUT"
+        env:
+          GH_TOKEN: ${{ github.token }}
+
+      - name: Construct devel docs
+        run: |
+          tmpdir=$(mktemp -dp "$RUNNER_TEMP" devel.XXXXXXXXXX)
+          # Get the name of the binary archive for the documentation target
+          release_archive=$(gh release download "$DEVEL" -p manifest.json -O - | ./release_manifest -f /dev/stdin get "$DOC_TARGET")
+          # Download the latest release binary
+          gh release download "$DEVEL" -p "$release_archive" -O "$tmpdir/$release_archive"
+          # Extract and remove the top-level directory
+          tar -C "$tmpdir" -xf "$tmpdir/$release_archive" --strip-components=1
+
+          mkdir -p built-docs
+          cp -rT "$tmpdir/doc/html" built-docs/devel
+          cp -rT "$tmpdir/doc/html" built-docs
+        env:
+          GH_TOKEN: ${{ github.token }}
+          DEVEL: ${{ steps.versions.outputs.devel }}
+
+      - uses: actions/upload-pages-artifact@v3
+        with:
+          path: built-docs/
+
+      - id: deploy
+        uses: actions/deploy-pages@v4

.github/workflows/publisher.yml

@@ -25,9 +25,6 @@ jobs:
       url: ${{ steps.release.outputs.url }}
 
     steps:
-      # Publish action needs a checkout
-      - uses: actions/checkout@v4
-
       - name: Obtain latest successful run id
         id: finder
         run: |
@@ -48,17 +45,9 @@ jobs:
           WORKFLOW: ci.yml
           CONCLUSION: success
           GH_TOKEN: ${{ github.token }}
+          GH_REPO: ${{ github.repository }}
 
       # Download the latest instance of artifacts from a build done previously
-      - name: Download generated docs
-        uses: actions/download-artifact@v4
-        with:
-          run-id: ${{ steps.finder.outputs.run_id }}
-          # Keep up-to-date with ci.yml
-          name: Generated docs
-          path: doc/html
-          github-token: ${{ github.token }}
-
       - name: Download generated source archive
         uses: actions/download-artifact@v4
         with:
@@ -87,12 +76,6 @@ jobs:
           path: release-staging
           github-token: ${{ github.token }}
 
-      - name: Publish docs
-        uses: JamesIves/[email protected]
-        with:
-          branch: gh-pages
-          folder: doc/html
-
       - id: release-files
         name: Create release manifest
         run: |

doc/ci.rst

@@ -96,8 +96,11 @@ was developed to leverage this.
 4. Assuming that the staging branch passes all tests, ``devel`` is
    fast-forwarded to the staging branch head commit.
 
-5. The "Publish" pipeline deploys the rendered documentation and binaries
-   generated by the testing pipeline earlier in the staging branch.
+5. The "Publish" pipeline deploys the binaries generated by the testing pipeline
+   earlier in the staging branch.
+
+6. The "Deploy documentation" pipeline deploys the latest documentation to GitHub
+   Pages.
 
 Assumptions in CI design
 ------------------------
@@ -161,9 +164,6 @@ which are used by other pipelines, these artifacts are:
 
 * "source archive": generated from the git clone.
 
-* "Generated docs": HTML documentation generated from source. This is the
-  ``doc/html`` folder after a ``koch docs`` run.
-
 Changes to how these artifacts are packaged must be reviewed carefully to
 ensure that dependent pipelines will still function.
 
@@ -245,6 +245,21 @@ Only one instance of this pipeline might be run at any given time, due to its
 mutating nature. Currently Github Action's ``concurrency`` feature is used to
 block multiple runs.
 
+"Deploy documentation"
+----------------------
+
+As the name suggests, this pipeline constructs and deploys the documentation
+website. This pipeline is dependent on the
+`Release manifest tool <#tools-release-manifest-tool>`_.
+
+Unlike other pipelines, the code run will always be that of the latest commit
+in ``devel`` branch. This means that the code should always be aware of
+differences between versions of |NimSkull| being deployed.
+
+Currently, only one instance of this pipeline might be run at any given time
+to prevent races. GitHub Actions's ``concurrency`` feature is used to
+serialize the runs.
+
 Development guidelines
 ======================
 

tools/release_manifest.nim

@@ -255,6 +255,23 @@ proc addCommand(manifest: string, archiveData: varargs[string]) =
   # Serialize a new manifest
   writeFile(manifest, $database.serialize())
 
+proc getCommand(manifest, target: string): int =
+  ## Implementation for the `get` subcommand.
+  ##
+  ## :manifest:
+  ##   The filename of the release manifest to inspect.
+  ##
+  ## :target:
+  ##   The triplet of interest.
+  let database = json.parseFile(manifest).deserialize()
+
+  let idx = database.triplet.find(target)
+  if idx < 0:
+    stderr.writeLine("error: target $1 could not be found in database")
+    return 1
+
+  stdout.writeLine(database.file[idx])
+
 func escapeDataForGithubActions(s: string): string =
   ## Escape the string `s` so that it can be used as data for workflow commands.
   # The list is obtained from here:
@@ -314,6 +331,7 @@ type
     Help = "help"
     Add = "add"
     FilesToUpload = "files-to-upload"
+    Get = "get"
     Version = "version"
 
   Flag {.pure.} = enum
@@ -363,6 +381,7 @@ Usage: $app <command> [args]...
 Commands:
   add              Add artifacts to the manifest
   files-to-upload  List the files to be uploaded
+  get              Get release artifact for a target
   version          Print the release version
   help             Display help for any subcommand
 
@@ -393,6 +412,15 @@ Options:
                                   format such that it can be used in workflow
                                   commands (ie. set-output) without losing data.
 
+$globalOpt
+"""
+
+  GetHelp = """
+Usage: $app get [options] [--] <target>
+
+Print the artifact file name of the given target triplet. An error will be raised
+if no artifact can be found for the given target.
+
 $globalOpt
 """
 
@@ -432,6 +460,8 @@ proc printHelp(action: Action) =
     stdout.write(AddHelp % defaultHelpFormat)
   of FilesToUpload:
     stdout.write(FilesToUploadHelp % defaultHelpFormat)
+  of Get:
+    stdout.write(GetHelp % defaultHelpFormat)
   of Version:
     stdout.write(VersionHelp % defaultHelpFormat)
 
@@ -504,6 +534,16 @@ proc dispatch(cli: Cli): int =
           result = 1
         else:
           filesToUploadCommand(manifest, format.get)
+      of Get:
+        let manifest = cli.flags.getOrDefault(Flag.File, DefaultManifestFile)
+        if cli.args.len == 1:
+          result = getCommand(manifest, cli.args[0])
+        else:
+          # No or more than one targets were given, print the help text and set failure.
+          if cli.args.len > 1:
+            stderr.writeLine("error: only one target is expected")
+          printHelp(cli.action)
+          result = 1
       of Version:
         let manifest = cli.flags.getOrDefault(Flag.File, DefaultManifestFile)
         versionCommand(manifest)