[CDAPI-100]: Integrate CI Tests with APIM Proxy III

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "name": "pathology-api-build-container",
-  "build":{
+  "build": {
     "dockerfile": "../infrastructure/images/build-container/Dockerfile",
     "args": {
       "INCLUDE_DEV_CERTS": "true"
@@ -38,13 +38,17 @@
       "omzPlugins": "https://github.com/zsh-users/zsh-autosuggestions.git https://github.com/zsh-users/zsh-syntax-highlighting.git"
     }
   },
+  "initializeCommand": "scripts/devcontainer/create-docker-network-if-required.sh && cp .tool-versions ${localWorkspaceFolder}/infrastructure/images/build-container/resources/.tool-versions && mkdir -p ${localEnv:HOME}/.proxygen",
   "postCreateCommand": "doas apk add --no-cache openjdk21-jre && scripts/devcontainer/configure-zsh.sh && bash -c 'source ~/.bashrc && make config && pyenv activate pathology && make dependencies'",
-  "initializeCommand": "scripts/devcontainer/create-docker-network-if-required.sh && cp .tool-versions ${localWorkspaceFolder}/infrastructure/images/build-container/resources/.tool-versions",
   "mounts": [
-    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind,consistency=cached"
+    "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind,consistency=cached",
+    "source=${localEnv:HOME}/.proxygen,target=/home/pathology-dev/.proxygen,type=bind,consistency=cached"
   ],
   "remoteUser": "pathology-dev",
   "containerUser": "pathology-dev",
+  "containerEnv": {
+    "REQUESTS_CA_BUNDLE": "/etc/ssl/cert.pem"
+  },
   "portsAttributes": {
     // Disabling automatic port forwarding as the devcontainer should already have access to any required ports.
     "default": {

.github/actions/create-lines-of-code-report/action.yaml

@@ -34,7 +34,7 @@ runs:
       run: zip lines-of-code-report.json.zip lines-of-code-report.json
     - name: "Upload CLOC report as an artefact"
       if: ${{ !env.ACT }}
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
       with:
         name: lines-of-code-report.json.zip
         path: ./lines-of-code-report.json.zip

.github/actions/run-test-suite/action.yaml

@@ -0,0 +1,58 @@
+name: "Run test suite"
+description: |
+  Composite action that runs a single test type (unit/contract/schema/integration/acceptance),
+  uploads the test artefacts, and publishes a JUnit summary to the GitHub Actions job.
+  Delegates to `make test-<type>` which invokes scripts/tests/run-test.sh.
+  For remote tests, pass apigee-access-token to authenticate with the APIM proxy.
+
+inputs:
+  test-type:
+    description: "Test type: unit, contract, schema, integration, acceptance"
+    required: true
+  apigee-access-token:
+    description: "Apigee access token (if needed)"
+    required: false
+    default: ""
+  env:
+    description: "Environment: local or remote"
+    required: false
+    default: "remote"
+
+runs:
+  using: composite
+  steps:
+    - name: "Run ${{ inputs.test-type }} tests"
+      shell: bash
+      env:
+        APIGEE_ACCESS_TOKEN: ${{ inputs.apigee-access-token }}
+        ENV: ${{ inputs.env }}
+      run: |
+        if [[ -n "${APIGEE_ACCESS_TOKEN}" ]]; then
+          echo "::add-mask::${APIGEE_ACCESS_TOKEN}"
+        fi
+        make test-${{ inputs.test-type }}
+
+    - name: "Upload ${{ inputs.test-type }} test results"
+      if: always()
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+      with:
+        name: ${{ inputs.test-type }}-test-results
+        path: pathology-api/test-artefacts/
+        retention-days: 30
+
+    - name: "Check ${{ inputs.test-type }}-tests.xml exists"
+      id: check
+      if: always()
+      shell: bash
+      run: |
+        if [[ -f "pathology-api/test-artefacts/${{ inputs.test-type }}-tests.xml" ]]; then
+          echo "exists=true" >> "$GITHUB_OUTPUT"
+        else
+          echo "exists=false" >> "$GITHUB_OUTPUT"
+        fi
+
+    - name: "Publish ${{ inputs.test-type }} test results to summary"
+      if: ${{ always() && steps.check.outputs.exists == 'true' }}
+      uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86
+      with:
+        paths: pathology-api/test-artefacts/${{ inputs.test-type }}-tests.xml

.github/actions/scan-dependencies/action.yaml

@@ -34,7 +34,7 @@ runs:
       run: zip sbom-repository-report.json.zip sbom-repository-report.json
     - name: "Upload SBOM report as an artefact"
       if: ${{ !env.ACT }}
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
       with:
         name: sbom-repository-report.json.zip
         path: ./sbom-repository-report.json.zip
@@ -51,7 +51,7 @@ runs:
       run: zip vulnerabilities-repository-report.json.zip vulnerabilities-repository-report.json
     - name: "Upload vulnerabilities report as an artefact"
       if: ${{ !env.ACT }}
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
       with:
         name: vulnerabilities-repository-report.json.zip
         path: ./vulnerabilities-repository-report.json.zip

.github/actions/setup-python-project/action.yaml

@@ -7,6 +7,11 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - name: "Install system dependencies"
+      shell: bash
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libxml2-dev libxslt-dev
     - name: "Set up Python"
       uses: actions/setup-python@v6
       with:

.github/actions/trivy-fs-sbom/action.yml

@@ -88,7 +88,7 @@ runs:
         github-pat: ${{ inputs.github-token }}
 
     - name: Upload SBOM artifact
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
       with:
         name: ${{ inputs.artifact-name }}
         path: sbom.spdx.json

.github/actions/trivy-fs-scan/action.yaml

@@ -65,7 +65,7 @@ runs:
         trivy-config: ${{ steps.trivy-config-check.outputs.config-arg }}
 
     - name: Upload Trivy Scan Artifact
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
       with:
         name: ${{ inputs.artifact-name }}
         path: trivy-fs-scan.json

.github/workflows/preview-env.yaml

@@ -25,6 +25,10 @@ env:
   PROXYGEN_KEY_ID: ${{ vars.PREVIEW_ENV_PROXYGEN_KEY_ID }}
   PROXYGEN_CLIENT_ID: ${{ vars.PREVIEW_ENV_PROXYGEN_CLIENT_ID }}
   PROXYGEN_API_NAME: ${{ vars.PROXYGEN_API_NAME }}
+  BASE_URL: 'https://internal-dev.api.service.nhs.uk/${{ vars.PROXYGEN_API_NAME }}-pr-${{ github.event.pull_request.number }}'
+  ENV: "remote"
+  PR_NUMBER: ${{ github.event.pull_request.number }}
+  HOST: "internal-dev.api.service.nhs.uk"
 
 jobs:
   pr-preview:
@@ -43,7 +47,7 @@ jobs:
         with:
           python-version: "${{ env.PYTHON_VERSION }}"
 
-      - name: "Setup Python project"
+      - name: Setup Python project
         uses: ./.github/actions/setup-python-project
         with:
           python-version: ${{ env.PYTHON_VERSION }}
@@ -403,6 +407,104 @@ jobs:
           proxygen-client-id: ${{ env.PROXYGEN_CLIENT_ID }}
           proxygen-api-name: ${{ env.PROXYGEN_API_NAME }}
 
+      - name: Retrieve Apigee Token
+        id: apigee-token
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          APIGEE_TOKEN="$(proxygen pytest-nhsd-apim get-token | jq -r '.pytest_nhsd_apim_token' 2>/dev/null)"
+          if [ -z "$APIGEE_TOKEN" ] || [ "$APIGEE_TOKEN" = "null" ]; then
+            echo "::error::Failed to retrieve Apigee token"
+            exit 1
+          fi
+
+          echo "::add-mask::$APIGEE_TOKEN"
+          printf 'apigee-access-token=%s\n' "$APIGEE_TOKEN" >> "$GITHUB_OUTPUT"
+          echo "Token retrieved successfully (length: ${#APIGEE_TOKEN})"
+
+      - name: "Create coverage artefact name"
+        id: create-name
+        uses: ./.github/actions/create-artefact-name
+        with:
+          prefix: coverage
+
+      # ---------- Test suites ----------
+      - name: "Run unit tests"
+        if: github.event.action != 'closed'
+        uses: ./.github/actions/run-test-suite
+        with:
+          test-type: unit
+          env: local
+
+      - name: "Run contract tests"
+        if: github.event.action != 'closed'
+        uses: ./.github/actions/run-test-suite
+        with:
+          test-type: contract
+          apigee-access-token: ${{ steps.apigee-token.outputs.apigee-access-token }}
+
+      - name: "Run schema validation tests"
+        if: github.event.action != 'closed'
+        uses: ./.github/actions/run-test-suite
+        with:
+          test-type: schema
+          apigee-access-token: ${{ steps.apigee-token.outputs.apigee-access-token }}
+
+      - name: "Run integration tests"
+        if: github.event.action != 'closed'
+        uses: ./.github/actions/run-test-suite
+        with:
+          test-type: integration
+          apigee-access-token: ${{ steps.apigee-token.outputs.apigee-access-token }}
+
+      - name: "Run acceptance tests"
+        if: github.event.action != 'closed'
+        uses: ./.github/actions/run-test-suite
+        with:
+          test-type: acceptance
+          apigee-access-token: ${{ steps.apigee-token.outputs.apigee-access-token }}
+
+      # ---------- Coverage & reporting ----------
+      - name: "Download all test coverage artefacts"
+        if: always() && github.event.action != 'closed'
+        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
+        with:
+          path: pathology-api/test-artefacts/
+          merge-multiple: false
+      - name: "Merge coverage data"
+        if: always() && github.event.action != 'closed'
+        run: make test-coverage
+      - name: "Rename coverage XML with unique name"
+        if: always() && github.event.action != 'closed'
+        run: |
+          cd pathology-api/test-artefacts
+          mv coverage-merged.xml "${{ steps.create-name.outputs.artefact-name }}.xml"
+      - name: "Upload combined coverage report"
+        if: always() && github.event.action != 'closed'
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: ${{ steps.create-name.outputs.artefact-name }}
+          path: pathology-api/test-artefacts
+          retention-days: 30
+
+      - name: "Download merged coverage report"
+        if: always() && github.event.action != 'closed'
+        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
+        with:
+          name: ${{ steps.create-name.outputs.artefact-name }}
+          path: coverage-reports/
+      - name: "SonarCloud Scan"
+        if: always() && github.event.action != 'closed'
+        uses: SonarSource/sonarqube-scan-action@a31c9398be7ace6bbfaf30c0bd5d415f843d45e9 #7.0.0
+        env:
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
+        with:
+          args: >
+            -Dsonar.organization=${{ vars.SONAR_ORGANISATION_KEY }}
+            -Dsonar.projectKey=${{ vars.SONAR_PROJECT_KEY }}
+            -Dsonar.python.coverage.reportPaths=coverage-reports/${{ steps.create-name.outputs.artefact-name }}.xml
+
       - name: Comment function name on PR
         if: github.event_name == 'pull_request' && github.event.action != 'closed'
         uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd
@@ -412,7 +514,7 @@ jobs:
             const mock_fn = '${{ steps.names.outputs.mock_function_name }}';
             const url = '${{ steps.names.outputs.preview_url }}';
             const mock_url = '${{ steps.names.outputs.mock_preview_url }}';
-            const proxy_url = 'https://internal-dev.api.service.nhs.uk/${{ env.PROXYGEN_API_NAME }}-pr-${{ github.event.pull_request.number }}';
+            const proxy_url = '${{ env.BASE_URL }}';
             const owner = context.repo.owner;
             const repo = context.repo.repo;
             const issueNumber = context.issue.number;