Push a test CSI driver image that can be used for operator testing

Author: iamjoemccormick

.github/workflows/build-test-publish-operator.yaml

@@ -1,34 +1,30 @@
-name: Build, Test, and Publish Operator Bundle
+name: "Test and Publish BeeGFS CSI Driver Operator"
 
+# Adapted from:
+# https://docs.github.com/en/actions/using-workflows/triggering-a-workflow
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#on
+# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#workflow_run
 on:
   workflow_dispatch:
-  push:
-    branches:
-      - "master"
-    tags:
-      - "v*"
-  pull_request:
-    branches:
-      - "master"
-
+  # This workflow should run after we've built the CSI driver container image and it is available as test package.
   workflow_run:
-    workflows: ["Build, Test, and Publish Container Images"]
+    workflows: [Test and Publish BeeGFS CSI Driver]
     types:
       - completed
 
 env:
-  # Container image registry to publish images to.
+  # Container image registry to publish images to:
   REGISTRY: ghcr.io
-  # Where to push an image of the operator that will be retained (for master builds or releases):
+  # Where to push an image of the operator that will be retained (for master builds or releases) without a specific tag:
   IMAGE_NAME: ghcr.io/thinkparq/beegfs-csi-driver-operator
   # Where to push an image of the operator for testing without a specific tag:
   TEST_IMAGE_NAME: ghcr.io/thinkparq/test-beegfs-csi-driver-operator
   # Where to push an image of the bundle for testing without a specific tag:
   TEST_BUNDLE_NAME: ghcr.io/thinkparq/test-beegfs-csi-driver-bundle
   # What container image should be deployed by the operator for testing the BeeGFS CSI driver deployment:
-  CSI_IMAGE_NAME: ghcr.io/thinkparq/beegfs-csi-driver
-  # TODO: Determine what tag we want to actually use for testing (latest or potentially the github.sha).
-  CSI_IMAGE_TAG: pr-2
+  TEST_CSI_IMAGE_NAME: ghcr.io/thinkparq/test-beegfs-csi-driver
+
+  # Note for all test images the github.sha will be used as the tag.
 
 jobs:
   build-and-unit-test:
@@ -104,7 +100,7 @@ jobs:
       fail-fast: true
       matrix:
         k8s-version: [1.26.3] # TODO: Expand tested versions (1.24.15, 1.25.11, 1.26.3, 1.27.3)
-        beegfs-version: [7.3.3] # TODO: Expand tested versions
+        beegfs-version: [7.3.4] # TODO: Expand tested versions
     permissions:
       packages: read
       contents: read
@@ -213,8 +209,8 @@ jobs:
 
       - name: Deploy a BeeGFSDriver object
         run: |
-          export CSI_IMAGE_NAME=$(echo ${{ env.CSI_IMAGE_NAME }})
-          export CSI_IMAGE_TAG=$(echo ${{ env.CSI_IMAGE_TAG }})
+          export CSI_IMAGE_NAME=$(echo ${{ env.TEST_CSI_IMAGE_NAME }})
+          export CSI_IMAGE_TAG=$(echo ${{ github.sha }})
           export BEEGFS_SECRET=$(echo ${{ secrets.CONN_AUTH_SECRET }})
           export BEEGFS_MGMTD=$(kubectl get nodes -o=jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')
           envsubst < test/env/beegfs-ubuntu/csi-beegfs-cr.yaml | kubectl apply -f -
@@ -268,25 +264,21 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 5
     needs: e2e-tests
-    # TODO: Consider if we want to continue publishing images on pull requests.
-    # if: github.event_name != 'pull_request'
+    if: github.event_name != 'pull_request'
     permissions:
       packages: write
       contents: read
     steps:
       - uses: actions/checkout@v3
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
       - name: Install Cosign
         uses: sigstore/[email protected]
         with:
           cosign-release: "v2.1.1"
 
       - name: Pull tested operator image from ghcr.io
         run: |
-          docker tag ${{ env.TEST_IMAGE_NAME }}:${{ github.sha }}
+          docker pull ${{ env.TEST_IMAGE_NAME }}:${{ github.sha }}
 
       - name: Log into the GitHub Container Registry
         uses: docker/login-action@v2
@@ -308,6 +300,7 @@ jobs:
             type=semver,pattern={{version}},prefix=v
             type=semver,pattern={{major}}.{{minor}},prefix=v
 
+      # TODO: Consider adding labels available as steps.meta.output.labels.
       - name: Tag and push the operator image to GitHub Container Registry
         run: |
           tags=$(echo "${{ steps.meta.outputs.tags }}" | tr '\n' ' ')
@@ -333,16 +326,12 @@ jobs:
         env:
           COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
           COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
-  # Test commands:
-  # make -e IMG=ghcr.io/thinkparq/test-beegfs-csi-driver-operator:master build docker-build docker-push
-  # make -e IMG=ghcr.io/thinkparq/test-beegfs-csi-driver-operator:master BUNDLE_IMG=ghcr.io/thinkparq/test-beegfs-csi-driver-bundle:master bundle bundle-build bundle-push
-  # operator-sdk run bundle ghcr.io/thinkparq/test-beegfs-csi-driver-bundle:master
 
   # We'll keep around a few old test packages to (a) avoid deleting image for workflows running in parallel,
   # and (b) it may be useful to pull a package to troubleshoot workflow failures.
   cleanup-test-images:
     runs-on: ubuntu-22.04
-    timeout-minutes: 5
+    timeout-minutes: 3
     needs: publish-images
     if: always()
     steps:

.github/workflows/build-test-publish.yaml

@@ -1,4 +1,4 @@
-name: Build, Test, and Publish Container Images
+name: "Test and Publish BeeGFS CSI Driver"
 
 on:
   workflow_dispatch:
@@ -12,10 +12,14 @@ on:
       - "master"
 
 env:
-  # Container image registry to publish images to.
+  # Container image registry to publish images to:
   REGISTRY: ghcr.io
-  # The name of the image without a specific tag.
+  # Where to push an image of the CSI driver that will be retained (for master builds or releases) without a specific tag:
   IMAGE_NAME: ghcr.io/thinkparq/beegfs-csi-driver
+  # Where to push an image of the CSI driver for testing (including the operator) without a specific tag:
+  TEST_IMAGE_NAME: ghcr.io/thinkparq/test-beegfs-csi-driver
+
+  # Note for all test images the github.sha will be used as the tag.
 
 jobs:
   build-and-unit-test:
@@ -57,35 +61,19 @@ jobs:
       # TODO: Can we cache anything here? test-vendor downloads a lot of stuff.
       # https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go#caching-dependencies
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      # For now just tag with the commit ID to ensure subsequent jobs in this workflow run use the correct image.
-      - name: Extract metadata for container image ${{ env.IMAGE_NAME }}
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: |
-            ${{ env.IMAGE_NAME }}
-          tags: |
-            type=sha,prefix=,format=long
-
-      # We already built the image but this allows us to easily apply the appropriate labels and export the image.
-      # It'll end up just reusing the local cache so it will complete very fast anyway.
-      - name: Build and export image ${{ env.IMAGE_NAME }}
-        uses: docker/build-push-action@v4
+      - name: Log into the GitHub Container Registry
+        uses: docker/login-action@v2
         with:
-          context: .
-          push: false
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          outputs: type=docker,dest=/tmp/beegfs-csi-driver.tar
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Upload the exported container image as an artifact for reuse in subsequent jobs
-        uses: actions/upload-artifact@v3
-        with:
-          name: beegfs-csi-driver-image
-          path: /tmp/beegfs-csi-driver.tar
+      # Push the image for reuse in subsequent steps, jobs, and workflows.
+      # For now just tag with the commit ID to ensure subsequent jobs in this workflow run use the correct image.
+      - name: Tag and push the CSI driver as a test package
+        run: |
+          docker tag beegfs-csi-driver:latest ${{ env.TEST_IMAGE_NAME }}:${{ github.sha }}
+          docker push ${{ env.TEST_IMAGE_NAME }}:${{ github.sha }}
 
   e2e-tests:
     runs-on: ubuntu-22.04
@@ -95,7 +83,7 @@ jobs:
       fail-fast: true
       matrix:
         k8s-version: [1.24.15, 1.25.11, 1.26.3, 1.27.3]
-        beegfs-version: [7.3.3] # TODO: Expand tested versions
+        beegfs-version: [7.3.4]
     permissions:
       packages: read
       contents: read
@@ -105,17 +93,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
-      - name: Download the container image artifact built in the previous job
-        uses: actions/download-artifact@v3
-        with:
-          name: beegfs-csi-driver-image
-          path: /tmp
-
-      - name: Load the downloaded container image into the local Docker cache
-        run: |
-          docker load --input /tmp/beegfs-csi-driver.tar
-          docker image ls -a
-
       - name: Deploy Kubernetes ${{ matrix.k8s-version }} using Minikube
         uses: medyagh/setup-minikube@latest
         with:
@@ -145,7 +122,7 @@ jobs:
           envsubst < test/env/beegfs-ubuntu/csi-beegfs-connauth.yaml > deploy/k8s/overlays/default/csi-beegfs-connauth.yaml
           # TODO: Enable once the K8s versions in the matrix are added to versions/
           # sed -i 's?/versions/latest?/versions/v${{ matrix.k8s-version }}?g' deploy/k8s/overlays/default/kustomization.yaml
-          echo -e "\nimages:\n  - name: ${{ env.IMAGE_NAME }}\n    newTag: ${{ github.sha }}" >> deploy/k8s/overlays/default/kustomization.yaml
+          echo -e "\nimages:\n  - name: ${{ env.IMAGE_NAME }}\n    newName: ${{ env.TEST_IMAGE_NAME }}\n    newTag: ${{ github.sha }}" >> deploy/k8s/overlays/default/kustomization.yaml
           kubectl apply -k deploy/k8s/overlays/default
 
       # TODO (BCSI-7): Actually run e2e tests using Ginko with an appropriate timeout.
@@ -197,7 +174,6 @@ jobs:
     runs-on: ubuntu-22.04
     timeout-minutes: 5
     needs: e2e-tests
-    # TODO: Consider if we want to continue publishing images on pull requests.
     if: github.event_name != 'pull_request'
     permissions:
       packages: write
@@ -213,16 +189,9 @@ jobs:
         with:
           cosign-release: "v2.1.1"
 
-      - name: Download BeeGFS CSI driver image artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: beegfs-csi-driver-image
-          path: /tmp
-
-      - name: Load image
+      - name: Pull tested CSI driver image from ghcr.io
         run: |
-          docker load --input /tmp/beegfs-csi-driver.tar
-          docker image ls -a
+          docker pull ${{ env.TEST_IMAGE_NAME }}:${{ github.sha }}
 
       - name: Log in to the GitHub Container Registry
         uses: docker/login-action@v2
@@ -244,11 +213,12 @@ jobs:
             type=semver,pattern={{version}},prefix=v
             type=semver,pattern={{major}}.{{minor}},prefix=v
 
+      # TODO: Consider adding labels available as steps.meta.output.labels.
       - name: Tag and push the image to GitHub Container Registry
         run: |
           tags=$(echo "${{ steps.meta.outputs.tags }}" | tr '\n' ' ')
           for tag in $tags; do
-            docker tag ${{ env.IMAGE_NAME }}:${{ github.sha }} $tag
+            docker tag ${{ env.TEST_IMAGE_NAME }}:${{ github.sha }} $tag
             docker push $tag
           done
 
@@ -269,3 +239,24 @@ jobs:
         env:
           COSIGN_PRIVATE_KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}
           COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
+
+  # We'll keep around a few old test packages to (a) avoid deleting image for workflows running in parallel,
+  # and (b) it may be useful to pull a package to troubleshoot workflow failures.
+  cleanup-test-images:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 3
+    needs: publish-images
+    if: always()
+    steps:
+      - name: Extract package names
+        run: |
+          test_image_name="${{ env.TEST_IMAGE_NAME }}"
+          test_image_pkg=${test_image_name##*/}
+          echo "TEST_IMAGE_PKG=$test_image_pkg" >> $GITHUB_ENV
+
+      - name: Cleanup old ${{ env.TEST_IMAGE_PKG }} packages
+        uses: actions/delete-package-versions@v4
+        with:
+          package-name: "${{ env.TEST_IMAGE_PKG }}"
+          package-type: "container"
+          min-versions-to-keep: 5

.github/workflows/codeql.yaml

@@ -9,7 +9,7 @@
 # the `language` matrix defined below to confirm you have the correct set of
 # supported CodeQL languages.
 #
-name: "CodeQL"
+name: "Code scanning using CodeQL"
 
 on:
   push:

Dockerfile

@@ -7,6 +7,9 @@ FROM gcr.io/distroless/static:latest
 
 LABEL maintainers="ThinkParQ"
 LABEL description="BeeGFS CSI Driver"
+LABEL org.opencontainers.image.description="BeeGFS CSI Driver"
+LABEL org.opencontainers.image.source="https://github.com/ThinkParQ/beegfs-csi-driver"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
 
 # Copy all built binaries to netapp/ directory.
 COPY bin/beegfs-csi-driver bin/chwrap netapp/

operator/Dockerfile

@@ -7,6 +7,9 @@
 # Use distroless as minimal base image to package the manager binary. Refer to
 # https://github.com/GoogleContainerTools/distroless for more details.
 FROM gcr.io/distroless/static:nonroot
+LABEL org.opencontainers.image.description="BeeGFS CSI Driver Operator"
+LABEL org.opencontainers.image.source="https://github.com/ThinkParQ/beegfs-csi-driver/operator"
+LABEL org.opencontainers.image.licenses="Apache-2.0"
 WORKDIR /
 COPY bin/manager .
 USER 65532:65532

operator/docs/developer-docs.md

@@ -323,10 +323,7 @@ Steps:
 
 1. In a terminal, navigate to the *operator/* directory.
 2. Set the IMAGE_TAG_BASE environment variable so that it refers to a
-   container registry namespace you have access to. For example, ThinkParQ
-   developers should execute `export
-   IMAGE_TAG_BASE=ghcr.io/thinkparq/test-beegfs-csi-driver-operator`.
-   External developers might execute `export
+   container registry namespace you have access to. For example, `export
    IMAGE_TAG_BASE=ghcr.io/thinkparq/test-beegfs-csi-driver-operator`.
 3. Set the VERSION environment variable. For example, execute
    `export VERSION=1.5.0`. The version MUST be semantic (e.g. 0.1.0) and