Fix verify_consumer_pacts GitHub action.

jladieu · jladieu · commit 0bce7579d921 · 2023-11-17T18:33:06.000-05:00
This is followup after: #1511 That PR (and associated changes) introduced the first WDS <-> WSM pact verification to WSM, but inadvertently broke the `verify_consumer_pacts` GitHub action in the process. This PR fixes that, while also adopting semantic versioning for reporting verification to Pact Broker. Various relevant threads: * https://broadinstitute.slack.com/archives/C043YJ40719/p1699888771297559 * https://broadinstitute.slack.com/archives/C043YJ40719/p1700155493619189 * https://broadinstitute.slack.com/archives/C043YJ40719/p1700154543175189 GitHub Actions logic copied from: DataBiosphere/terra-billing-profile-manager#324
diff --git a/.github/workflows/consumer_contract_tests.yml b/.github/workflows/consumer_contract_tests.yml
@@ -21,9 +21,55 @@ on:
     branches: [ main ]
     paths-ignore: [ '**.md' ]
 
+env:
+  PUBLISH_CONTRACTS_RUN_NAME: 'publish-contracts-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
+  CAN_I_DEPLOY_RUN_NAME: 'can-i-deploy-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
+
 jobs:
+  bump-check:
+    runs-on: ubuntu-latest
+    outputs:
+      is-bump: ${{ steps.skiptest.outputs.is-bump }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Skip version bump merges
+        id: skiptest
+        uses: ./.github/actions/bump-skip
+        with:
+          event-name: ${{ github.event_name }}
+
+# The primary objective of this section is to carefully control the dispatching of tags,
+# ensuring it only occurs during the 'Tag, publish, deploy' workflow.
+# However, a challenge arises with contract tests, as they require knowledge of the upcoming tag
+# before the actual deployment. To address this, we leverage the dry run feature provided by bumper.
+# This allows us to obtain the next tag for publishing contracts and verifying consumer pacts without
+# triggering the tag dispatch. This approach sidesteps the need for orchestrating multiple workflows,
+# simplifying our implementation.
+#
+# We regulate the tag job to meet the following requirements according to the trigger event type:
+# 1. pull_request event (due to opening or updating of PR branch):
+#      dry-run flag is set to false
+#         this allows the new semver tag #major.#minor.#patch-#commit to be used to identity pacticipant version for development purpose
+#         PR has no effect on the value of the latest tag in settings.gradle on disk
+# 2. PR merge to main, this triggers a push event on the main branch:
+#      dry-run flag is set to true
+#         this allows the new semver tag #major.#minor.#patch to be used to identity pacticipant version, and
+#         this action will not update the value of the latest tag in settings.gradle on disk
+#
+# Note: All workflows from the same PR merge should have the same copy of settings.gradle on disk,
+# which should be the one from the HEAD of the main branch before the workflow starts running
+  regulated-tag-job:
+    needs: [ bump-check ]
+    if: ${{ needs.bump-check.outputs.is-bump == 'no' }}
+    uses: ./.github/workflows/tag.yml
+    with:
+      dry-run: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+    secrets: inherit
+
   init-github-context:
     runs-on: ubuntu-latest
+    needs: [ bump-check ]
+    if: ${{ needs.bump-check.outputs.is-bump == 'no' }}
     outputs:
       repo-branch: ${{ steps.extract-branch.outputs.repo-branch }}
       repo-version: ${{ steps.extract-branch.outputs.repo-version }}
@@ -56,7 +102,8 @@ jobs:
 
   wsm-consumer-contract-tests:
     runs-on: ubuntu-latest
-    needs: [ init-github-context ]
+    needs: [ bump-check, init-github-context ]
+    if: ${{ needs.bump-check.outputs.is-bump == 'no' }}
     outputs:
       pact-b64: ${{ steps.encode-pact.outputs.pact-b64 }}
 
@@ -77,15 +124,22 @@ jobs:
 
   publish-contracts:
     runs-on: ubuntu-latest
-    needs: [ init-github-context, wsm-consumer-contract-tests ]
+    needs: [ bump-check, init-github-context, wsm-consumer-contract-tests, regulated-tag-job ]
+    if: ${{ needs.bump-check.outputs.is-bump == 'no' }}
     steps:
       - name: Dispatch to terra-github-workflows
-        uses: broadinstitute/workflow-dispatch@v3
+        uses: broadinstitute/workflow-dispatch@v4.0.0
         with:
+          run-name: "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}"
           workflow: .github/workflows/publish-contracts.yaml
           repo: broadinstitute/terra-github-workflows
           ref: refs/heads/main
           token: ${{ secrets.BROADBOT_TOKEN }} # github token for access to kick off a job in the private repo
-          inputs: '{ "pact-b64": "${{ needs.wsm-consumer-contract-tests.outputs.pact-b64 }}", "repo-owner": "${{ github.repository_owner }}", "repo-name": "${{ github.event.repository.name }}", "repo-branch": "${{ needs.init-github-context.outputs.repo-branch }}" }'
-
-
+          inputs: '{
+            "run-name": "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}",
+            "pact-b64": "${{ needs.wsm-consumer-contract-tests.outputs.pact-b64 }}",
+            "repo-owner": "${{ github.repository_owner }}",
+            "repo-name": "${{ github.event.repository.name }}",
+            "repo-branch": "${{ needs.init-github-context.outputs.repo-branch }}",
+            "release-tag": "${{ needs.regulated-tag-job.outputs.new-tag }}"
+          }'
diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml
@@ -0,0 +1,61 @@
+name: Tag
+on:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      dry-run:
+        description: "Determine the next version without tagging the branch. The workflow can use the outputs new_tag and tag in subsequent steps. Possible values are true and false (default)"
+        default: false
+        required: false
+        type: string
+      print-tag:
+        description: "Echo tag to console"
+        default: true
+        required: false
+        type: string
+    outputs:
+      tag:
+        description: "The value of the latest tag after running this action"
+        value: ${{ jobs.tag-job.outputs.tag }}
+      new-tag:
+        description: "The value of the newly created tag"
+        value: ${{ jobs.tag-job.outputs.new-tag }}
+    secrets:
+      BROADBOT_TOKEN:
+        required: true
+
+jobs:
+# On tag vs. new-tag.
+# The new-tag is always the tag resulting from a bump to the original tag.
+# However, the tag is by definition the value of the latest tag after running the action,
+# which might not change if dry run is used, and remains same as the original tag.
+  tag-job:
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
+      new-tag: ${{ steps.tag.outputs.new_tag }}
+    steps:
+      - name: Checkout current code
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.BROADBOT_TOKEN }} # this allows the push to succeed later
+      - name: Bump the tag to a new version
+        # https://github.com/DataBiosphere/github-actions/tree/master/actions/bumper
+        uses: databiosphere/github-actions/actions/bumper@bumper-0.0.6
+        id: tag
+        env:
+          GITHUB_TOKEN: ${{ secrets.BROADBOT_TOKEN }}
+          HOTFIX_BRANCHES: hotfix.*
+          DEFAULT_BUMP: patch
+          DRY_RUN: ${{ inputs.dry-run }}
+          RELEASE_BRANCHES: main
+          VERSION_FILE_PATH: settings.gradle
+          VERSION_LINE_MATCH: "^\\s*gradle.ext.releaseVersion\\s*=\\s*'.*'"
+          VERSION_SUFFIX: SNAPSHOT
+      - name: Echo tag to console
+        if: ${{ inputs.print-tag == 'true' }}
+        run: |
+          echo "Newly created version tag: '${{ steps.tag.outputs.new_tag }}'"
+          echo "settings.gradle"
+          echo "==============="
+          cat settings.gradle
diff --git a/.github/workflows/verify_consumer_pacts.yaml b/.github/workflows/verify_consumer_pacts.yaml
@@ -14,21 +14,16 @@ name: Verify consumer pacts
 # The workflow requires the following Pact Broker credentials:
 # - PACT_BROKER_USERNAME - the Pact Broker username
 # - PACT_BROKER_PASSWORD - the Pact Broker password
-# They are managed by Atlantis and were added to Terraform here:
-# https://github.com/broadinstitute/terraform-ap-deployments/pull/1086
-env:
-  spring_profiles_active: human-readable-logging
 on:
   pull_request:
-    branches:
-      - develop
-    paths-ignore:
-      - 'README.md'
+    branches: [ main ]
+    paths-ignore: [ '**.md' ]
   push:
-    branches:
-      - develop
-    paths-ignore:
-      - 'README.md'
+    branches: [ main ]
+    paths-ignore: [ '**.md' ]
+  merge_group:
+    branches: [ main ]
+    paths-ignore: [ '**.md' ]
   workflow_dispatch:
     inputs:
       pb-event-type:
@@ -72,14 +67,44 @@ on:
         required: true
         type: string
 
+env:
+  spring_profiles_active: human-readable-logging
+  CAN_I_DEPLOY_RUN_NAME: 'can-i-deploy-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}'
+
 jobs:
+  bump-check:
+    runs-on: ubuntu-latest
+    outputs:
+      is-bump: ${{ steps.skiptest.outputs.is-bump }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Skip version bump merges
+        id: skiptest
+        uses: ./.github/actions/bump-skip
+        with:
+          event-name: ${{ github.event_name }}
+
+  # We only need a new version tag when this workflow is triggered by opening, updating a PR or PR merge.
+  # When triggered by a Pact Broker webhook, the provider version (GIT hash or release tag)
+  # is already included in the payload, then a new version tag wouldn't be needed.
+  regulated-tag-job:
+    needs: [ bump-check ]
+    if: ${{ needs.bump-check.outputs.is-bump == 'no' }}
+    uses: ./.github/workflows/tag.yml
+    with:
+      dry-run: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+    secrets: inherit
+
   verify-consumer-pact:
+    needs: [ bump-check, regulated-tag-job ]
+    if: ${{ needs.bump-check.outputs.is-bump == 'no' }}
     runs-on: ubuntu-latest
     permissions:
       contents: 'read'
       id-token: 'write'
     outputs:
       provider-sha: ${{ steps.verification-test.outputs.provider-sha }}
+      provider-version: ${{ steps.verification-test.outputs.provider-version }}
 
     steps:
       - name: Checkout current code
@@ -90,20 +115,23 @@ jobs:
       - name: Extract branch
         id: extract-branch
         run: |
-          GITHUB_EVENT_NAME=${{ github.event_name }}
-          if [[ "$GITHUB_EVENT_NAME" == "push" || "$GITHUB_EVENT_NAME" == "workflow_dispatch"]]; then
-            GITHUB_REF=${{ github.ref }} # The Git Ref that this workflow runs on
-            GITHUB_SHA=${{ github.sha }} # The Git Sha that this workflow runs on
+          if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then
+            GITHUB_REF=${{ github.ref }}
+            GITHUB_SHA=${{ github.sha }}
           elif [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
             GITHUB_REF=refs/heads/${{ github.head_ref }}
             GITHUB_SHA=${{ github.event.pull_request.head.sha }}
+          elif [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]]; then
+            GITHUB_REF=${{ github.ref }} # The Git Ref that this workflow runs on
+            GITHUB_SHA=${{ github.sha }} # The Git Sha that this workflow runs on
           else
             echo "Failed to extract branch information"
             exit 1
           fi
           echo "CURRENT_BRANCH=${GITHUB_REF/refs\/heads\//""}" >> $GITHUB_ENV
           echo "CURRENT_SHA=$GITHUB_SHA" >> $GITHUB_ENV
-      - name: "This step will only run when this workflow is triggered by a Pact Broker webhook event"
+
+      - name: Set environment variables when triggered by a Pact Broker webhook event
         if: ${{ inputs.pb-event-type != '' }}
         run: |
           echo "pb-event-type=${{ inputs.pb-event-type }}"
@@ -124,6 +152,15 @@ jobs:
           echo "CONSUMER_NAME=${{ inputs.consumer-name }}" >> $GITHUB_ENV
           echo "CONSUMER_BRANCH=${{ inputs.consumer-version-branch }}" >> $GITHUB_ENV
           echo "CONSUMER_SHA=${{ inputs.consumer-version-number }}" >> $GITHUB_ENV
+
+      - name: Set provider version envvar
+        run: |
+          if [[ -z "${{ inputs.pb-event-type }}" ]]; then
+            echo "PROVIDER_VERSION=${{ needs.regulated-tag-job.outputs.new-tag }}" >> $GITHUB_ENV
+          else
+            echo "PROVIDER_VERSION=${{ env.PROVIDER_SHA }}" >> $GITHUB_ENV
+          fi
+
       - name: Switch to appropriate branch
         run: |
           if [[ -z "${{ env.PROVIDER_BRANCH }}" ]]; then
@@ -149,6 +186,7 @@ jobs:
           fi
           echo "git rev-parse HEAD"
           git rev-parse HEAD
+
       - name: Set up JDK 17
         uses: actions/setup-java@v2
         with:
@@ -167,12 +205,15 @@ jobs:
       - name: Verify consumer pacts and publish verification status to Pact Broker
         id: verification-test
         env:
+          PACT_BROKER_URL: https://pact-broker.dsp-eng-tools.broadinstitute.org
           PACT_PROVIDER_COMMIT: ${{ env.PROVIDER_SHA }}
           PACT_PROVIDER_BRANCH: ${{ env.PROVIDER_BRANCH }}
           PACT_BROKER_USERNAME: ${{ secrets.PACT_BROKER_USERNAME }}
           PACT_BROKER_PASSWORD: ${{ secrets.PACT_BROKER_PASSWORD }}
+          PACT_PROVIDER_VERSION: ${{ env.PROVIDER_VERSION }}
         run: |
           echo "provider-sha=${{ env.PROVIDER_SHA }}" >> $GITHUB_OUTPUT
+          echo "provider-version=${{ env.PACT_PROVIDER_VERSION }}" >> $GITHUB_OUTPUT
           echo "env.CHECKOUT_BRANCH=${{ env.CHECKOUT_BRANCH }} # If not empty, this reflects the branch being checked out (generated by Pact Broker)"
           echo "env.CHECKOUT_SHA=${{ env.CHECKOUT_SHA }}       # If not empty, this reflects the git commit hash of the branch being checked out (generated by Pact Broker)"
           echo "env.CURRENT_BRANCH=${{ env.CURRENT_BRANCH }}   # This reflects the branch being checked out if CHECKOUT_BRANCH is empty"
@@ -181,6 +222,7 @@ jobs:
           echo "env.PROVIDER_SHA=${{ env.PROVIDER_SHA }}       # This reflects the provider version for pact verification"
           echo "env.CONSUMER_BRANCH=${{ env.CONSUMER_BRANCH }} # This reflects the consumer branch for pact verification (generated by Pact Broker)"
           echo "env.CONSUMER_SHA=${{ env.CONSUMER_SHA }}       # This reflects the consumer version for pact verification (generated by Pact Broker)"
+          echo "env.PACT_PROVIDER_VERSION=${{ env.PACT_PROVIDER_VERSION }}  # Deprecate env.PACT_PROVIDER_COMMIT. This new envvar is used for migrating GIT hash to app versioning"
           ./gradlew --build-cache verifyPacts --scan
       - name: Upload Test Reports
         if: always()
@@ -193,14 +235,19 @@ jobs:
     # The can-i-deploy job will run as a result of a workspace manager PR.
     # It reports the pact verification statuses on all deployed environments.
     runs-on: ubuntu-latest
-    if: ${{ inputs.pb-event-type == '' }}
-    needs: [ verify-consumer-pact ]
+    if: ${{ inputs.pb-event-type == '' && needs.bump-check.outputs.is-bump == 'no' }}
+    needs: [ verify-consumer-pact, bump-check ]
     steps:
       - name: Dispatch to terra-github-workflows
-        uses: broadinstitute/workflow-dispatch@v3
+        uses: broadinstitute/workflow-dispatch@v4.0.0
         with:
+          run-name: "${{ env.CAN_I_DEPLOY_RUN_NAME }}"
           workflow: .github/workflows/can-i-deploy.yaml
           repo: broadinstitute/terra-github-workflows
           ref: refs/heads/main
           token: ${{ secrets.BROADBOT_TOKEN }} # github token for access to kick off a job in the private repo
-          inputs: '{ "pacticipant": "workspacemanager", "version": "${{ needs.verify-consumer-pact.outputs.provider-sha }}" }'
+          inputs: '{
+            "run-name": "${{ env.CAN_I_DEPLOY_RUN_NAME }}",
+            "pacticipant": "workspacemanager",
+            "version": "${{ needs.verify-consumer-pact.outputs.provider-sha }}"
+          }'
diff --git a/service/src/test/java/bio/terra/workspace/pact/WdsContractVerificationTest.java b/service/src/test/java/bio/terra/workspace/pact/WdsContractVerificationTest.java
@@ -28,6 +28,7 @@
 import bio.terra.workspace.db.WorkspaceActivityLogDao;
 import bio.terra.workspace.db.WorkspaceDao;
 import bio.terra.workspace.db.model.DbCloudContext;
+import bio.terra.workspace.service.danglingresource.DanglingResourceCleanupService;
 import bio.terra.workspace.service.iam.AuthHeaderKeys;
 import bio.terra.workspace.service.iam.AuthenticatedUserRequest;
 import bio.terra.workspace.service.iam.AuthenticatedUserRequestFactory;
@@ -87,6 +88,9 @@ public class WdsContractVerificationTest extends BaseUnitTestMocks {
   @MockBean(name = "postSetupInitialization")
   private SmartInitializingSingleton unusedSmartInitializingSingleton;
 
+  // Mocked out to prevent error trying to clean up nonexistent tables
+  @MockBean private DanglingResourceCleanupService unusedDanglingResourceCleanupService;
+
   @MockBean private WorkspaceActivityLogDao unusedWorkspaceActivityLogDao;
 
   // needed to mock behavior in this test
