CI: Add SBOM generation to container image builds

.github/workflows/stackhpc-container-image-build.yml

@@ -38,7 +38,12 @@ on:
         type: boolean
         required: false
         default: true
-      push-dirty:
+      sbom:
+        description: Generate SBOM?
+        type: boolean
+        required: false
+        default: true
+      push-critical:
         description: Push scanned images that have critical vulnerabilities?
         type: boolean
         required: false
@@ -158,7 +163,7 @@ jobs:
 
       - name: Install Trivy
         run: |
-          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0
+          curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.62.1
 
       - name: Install yq
         run: |
@@ -252,14 +257,14 @@ jobs:
         run: if [ $(wc -l < ${{ matrix.distro.name }}-${{ matrix.distro.release }}-container-images) -le 1 ]; then exit 1; fi
 
       - name: Scan built container images
-        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro.name }}-${{ matrix.distro.release }} ${{ steps.write-kolla-tag.outputs.kolla-tag }}
+        run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro.name }}-${{ matrix.distro.release }} ${{ steps.write-kolla-tag.outputs.kolla-tag }} ${{ inputs.sbom && '--sbom'}}
 
       - name: Move image scan logs to output artifact
         run: mv image-scan-output image-build-logs/image-scan-output
 
-      - name: Fail if no images have passed scanning
+      - name: Fail if any images have critical vulnerabilities
         run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi
-        if: ${{ !inputs.push-dirty }}
+        if: ${{ !inputs.push-critical }}
 
       - name: Copy clean images to push-attempt-images list
         run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt
@@ -269,13 +274,13 @@ jobs:
       # This should be reverted when it's decided to filter high level CVEs as well.
       - name: Append dirty images to push list
         run: |
-          cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt
+          cat image-build-logs/image-scan-output/high-images.txt >> image-build-logs/push-attempt-images.txt
         if: ${{ inputs.push }}
 
       - name: Append images with critical vulnerabilities to push list
         run: |
           cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt
-        if: ${{ inputs.push && inputs.push-dirty }}
+        if: ${{ inputs.push && inputs.push-critical }}
 
       - name: Push images
         run: |
@@ -324,12 +329,12 @@ jobs:
       # This can be used again instead of "Fail when critical vulnerabilities are found" when it's
       # decided to fail the job on detecting high CVEs as well.
       # - name: Fail when images failed scanning
-      #   run: if [ $(wc -l < image-build-logs/image-scan-output/dirty-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/dirty-images.txt && exit 1; fi
-      #   if: ${{ !inputs.push-dirty && !cancelled() }}
+      #   run: if [ $(wc -l < image-build-logs/image-scan-output/high-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/high-images.txt && exit 1; fi
+      #   if: ${{ !inputs.push-critical && !cancelled() }}
 
       - name: Fail when critical vulnerabilities are found
         run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi
-        if: ${{ !inputs.push-dirty && !cancelled() }}
+        if: ${{ !inputs.push-critical && !cancelled() }}
 
       # NOTE(mgoddard): Trigger another CI workflow in the
       # stackhpc-release-train repository.

tools/scan-images.sh

@@ -1,102 +1,168 @@
 #!/usr/bin/env bash
 set -eo pipefail
 
-# Check correct usage
-if [[ ! $2 ]]; then
-  echo "Usage: scan-images.sh <os-distribution> <image-tag>"
-  exit 2
-fi
-
-set -u
+# Global variables
+scan_common_args=" \
+                  --exit-code 1 \
+                  --scanners vuln \
+                  --format json \
+                  --severity HIGH,CRITICAL \
+                  --ignore-unfixed \
+                  --db-repository ghcr.io/aquasecurity/trivy-db:2 \
+                  --db-repository public.ecr.aws/aquasecurity/trivy-db \
+                  --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \
+                  --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db "
 
-# Check that trivy is installed
-if ! trivy --version; then
-  echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin v0.49.1'
-fi
-
-# Clear any previous outputs
-rm -rf image-scan-output
+# Print usage instructions and error with wrong inputs
+usage() {
+  echo "Usage: scan-images.sh <os-distribution> <image-tag> [--sbom]"
+  exit 2
+}
 
-# Make a fresh output directory
-mkdir -p image-scan-output
+# Check dependencies are installed, print installation instructions otherwise
+check_deps_installed() {
+  if ! trivy --version > /dev/null; then
+    echo 'Please install trivy: curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.62.1'
+    exit 1
+  fi
+  if ! yq --version > /dev/null; then
+    echo 'Please install yq: sudo dnf/apt install yq'
+    exit 1
+  fi
+}
 
-# Get built container images
-docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2" > $1-scanned-container-images.txt
+# Prepare output files
+file_prep() {
+  rm -rf image-scan-output
+  mkdir -p image-scan-output
+  touch image-scan-output/clean-images.txt image-scan-output/high-images.txt image-scan-output/critical-images.txt
+}
 
-# Make a file of imagename:tag
-images=$(grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:)
+# Gather image lists
+get_images() {
+  docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:$2" > $1-scanned-container-images.txt
+  grep --invert-match --no-filename ^REPOSITORY $1-scanned-container-images.txt | sed 's/ \+/:/g' | cut -f 1,2 -d:
+}
 
-# Ensure output files exist
-touch image-scan-output/clean-images.txt image-scan-output/dirty-images.txt image-scan-output/critical-images.txt
+# Generate ignored vulnerabilities file
+generate_trivy_ignore() {
+  local imagename=$1
+  local global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
+  local image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml 2> /dev/null)
 
-# If Trivy detects no vulnerabilities, add the image name to clean-images.txt.
-# If there are vulnerabilities detected, add it to dirty-images.txt and
-# generate a csv summary
-# If the image contains at least one critical vulnerabilities, add it to
-# critical-images.txt
-for image in $images; do
-  filename=$(basename $image | sed 's/:/\./g')
-  imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g')
-  global_vulnerabilities=$(yq .global_allowed_vulnerabilities[] src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml)
-  image_vulnerabilities=$(yq .$imagename'_allowed_vulnerabilities[]' src/kayobe-config/etc/kayobe/trivy/allowed-vulnerabilities.yml)
   touch .trivyignore
   for vulnerability in $global_vulnerabilities; do
     echo $vulnerability >> .trivyignore
   done
   for vulnerability in $image_vulnerabilities; do
     echo $vulnerability >> .trivyignore
   done
-  if $(trivy image \
-          --quiet \
-          --exit-code 1 \
-          --scanners vuln \
-          --format json \
-          --severity HIGH,CRITICAL \
-          --output image-scan-output/${filename}.json \
-          --ignore-unfixed \
-          --db-repository ghcr.io/aquasecurity/trivy-db:2 \
-          --db-repository public.ecr.aws/aquasecurity/trivy-db \
-          --java-db-repository ghcr.io/aquasecurity/trivy-java-db:1 \
-          --java-db-repository public.ecr.aws/aquasecurity/trivy-java-db \
-          $image); then
-    # Clean up the output file for any images with no vulnerabilities
-    rm -f image-scan-output/${filename}.json
-
-    # Add the image to the clean list
+}
+
+# Put results into CSV
+generate_summary_csv() {
+  local imagename=$1
+  local filename=$2
+
+  echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${imagename}/${filename}-summary.csv
+
+  jq -r '.Results[]
+      | select(.Vulnerabilities)
+      | .Vulnerabilities
+      | map(select(.PkgName | test("kernel") | not ))
+      | group_by(.VulnerabilityID)
+      | map(
+        [
+          (map(.PkgName) | unique | join(";")),
+          (map(.PkgPath | select( . != null )) | join(";")),
+          .[0].PkgID,
+          .[0].VulnerabilityID,
+          .[0].FixedVersion,
+          .[0].PrimaryURL,
+          .[0].Severity
+          ]
+        )
+      | .[]
+      | @csv' image-scan-output/${imagename}/${filename}-scan.json >> image-scan-output/${imagename}/${filename}-summary.csv
+}
+
+# Categorise images based on severity
+categorise_image() {
+  local imagename=$1
+  local filename=$2
+  local image=$3
+
+  if [ $(grep "CRITICAL" image-scan-output/${imagename}/${filename}-summary.csv -c) -gt 0 ]; then
+    echo "${image}" >> image-scan-output/critical-images.txt
+  else
+    echo "${image}" >> image-scan-output/high-images.txt
+  fi
+}
+
+# Generate SBOM, return correct scan command for SBOM
+generate_sbom() {
+  local imagename=$1
+  local filename=$2
+  local image=$3
+  trivy image \
+        --format spdx-json \
+        --output image-scan-output/${imagename}/${filename}-sbom.json \
+        $image > /dev/null 2>&1
+  echo "trivy sbom $scan_common_args \
+        --output image-scan-output/${imagename}/${filename}-scan.json \
+        image-scan-output/${imagename}/${filename}-sbom.json"
+}
+
+# Scan images, generate SBOMs if requested
+scan_image() {
+  local image=$1
+  local filename=$(basename $image | sed 's/:/\./g')
+  local imagename=$(echo $filename | cut -d "." -f 1 | sed 's/-/_/g')
+
+  mkdir -p image-scan-output/$imagename
+  generate_trivy_ignore $imagename
+
+  # If SBOM is required, generate it first and scan the results, otherwise we
+  # scan the image directly.
+  if $generate_sbom; then
+    echo "Generating SBOM for $imagename"
+    scan_command=$(generate_sbom $imagename $filename $image)
+  else
+    scan_command="trivy image $scan_common_args \
+                  --output image-scan-output/${imagename}/${filename}-scan.json $image"
+  fi
+
+  # Run scan against image or SBOM, format output. If no results, delete files.
+  echo "Scanning $imagename for vulnerabilities"
+  if $scan_command > /dev/null 2>&1; then
+    rm -f image-scan-output/${imagename}/${filename}-scan.json
     echo "${image}" >> image-scan-output/clean-images.txt
   else
+    generate_summary_csv $imagename $filename
+    categorise_image $imagename $filename $image
+  fi
+}
 
-    # Write a header for the summary CSV
-    echo '"PkgName","PkgPath","PkgID","VulnerabilityID","FixedVersion","PrimaryURL","Severity"' > image-scan-output/${filename}.summary.csv
-
-    # Write the summary CSV data
-    jq -r '.Results[]
-            | select(.Vulnerabilities)
-            | .Vulnerabilities
-            # Ignore packages with "kernel" in the PkgName
-            | map(select(.PkgName | test("kernel") | not ))
-            | group_by(.VulnerabilityID)
-            | map(
-                  [
-                    (map(.PkgName) | unique | join(";")),
-                    (map(.PkgPath | select( . != null )) | join(";")),
-                    .[0].PkgID,
-                    .[0].VulnerabilityID,
-                    .[0].FixedVersion,
-                    .[0].PrimaryURL,
-                    .[0].Severity
-                    ]
-                )
-            | .[]
-            | @csv' image-scan-output/${filename}.json >> image-scan-output/${filename}.summary.csv
-
-    if [ $(grep "CRITICAL" image-scan-output/${filename}.summary.csv -c) -gt 0 ]; then
-      # If the image contains critical vulnerabilities, add the image to critical list
-      echo "${image}" >> image-scan-output/critical-images.txt
-    else
-      # Otherwise, add the image to the dirty list
-      echo "${image}" >> image-scan-output/dirty-images.txt
-    fi
+# Main function
+main() {
+  if [[ ! $2 ]]; then
+    usage
   fi
-  rm .trivyignore
-done
+
+  generate_sbom=false
+  if [[ "$3" == "--sbom" ]]; then
+    generate_sbom=true
+  fi
+
+  set -u
+
+  check_deps_installed
+  file_prep
+
+  images=$(get_images $1 $2)
+  for image in $images; do
+    scan_image $image
+  done
+}
+
+main "$@"