test: Repurpose ADO release pipeline as integration testing pipeline

.azure-devops/create-release.yml → .azure-devops/integration_tests.yml

@@ -7,23 +7,17 @@ parameters:
 - name: variableGroup
   type: string
   default: 'aziotcli_test_primary'
-- name: buildAgentPoolVar
-  type: string
-  default: 'BuildAgentPool'
-- name: buildAgentVmImageVar
-  type: string
-  default: 'BuildAgentVmImage'
 - name: testAgentVmImage
   type: string
-  default: 'ubuntu-20.04'
+  default: 'ubuntu-22.04'
   values:
   - 'ubuntu-20.04'
   - 'ubuntu-22.04'
   - 'ubuntu-latest'
 - name: pythonVersion
   displayName: 'Python version for building wheel, KPIs'
   type: string
-  default: '3.8'
+  default: '3.9'
   values:
   - '3.8'
   - '3.9'
@@ -69,10 +63,6 @@ parameters:
 
 variables:
   - group: ${{ parameters.variableGroup }}
-  - name: vmImage
-    value: $[variables.${{ parameters.buildAgentVmImageVar }}]
-  - name: buildPool
-    value: $[variables.${{ parameters.buildAgentPoolVar }}]
 
 stages:
   - stage: 'build'
@@ -81,10 +71,7 @@ stages:
 
     - job: 'Build_Publish_Azure_IoT_CLI_Extension'
       pool:
-        name: $(buildPool)
-        vmImage: $(vmImage)
-        demands:
-        - ImageOverride -equals $(vmImage)
+        vmImage: ubuntu-latest
       steps:
       - task: UsePythonVersion@0
         inputs:
@@ -148,7 +135,6 @@ stages:
     #   displayName: 'Validate Reference Document Generation'
     #   steps:
     #   - template: templates/validate-refdoc-generation.yml
-
   - stage: 'smokeTest'
     displayName: 'Run smoke tests'
     dependsOn: test
@@ -172,62 +158,4 @@ stages:
       - template: templates/calculate-code-coverage.yml
         parameters:
           pythonVersion: ${{ parameters.pythonVersion }}
-          architecture: ${{ parameters.architecture }}
-
-  - stage: 'release'
-    displayName: 'Stage GitHub release'
-    dependsOn: build
-    condition: and(succeeded(), eq(${{ parameters.stageForPublish }}, 'true'))
-    jobs:
-    - deployment: 'StageGitHub'
-      displayName: 'Stage CLI extension on GitHub'
-      environment: 'production'
-
-    - job: 'Calculate_Sha_And_Create_Release'
-      displayName: 'Calculate package hash and publish'
-      pool:
-        name: $(buildPool)
-        vmImage: $(vmImage)
-        demands:
-        - ImageOverride -equals $(vmImage)
-      variables:
-        CLIVersion: $[ stageDependencies.build.recordVersion.outputs['setupVersion.CLIVersion'] ]
-        ReleaseTitle: $[ stageDependencies.build.recordVersion.outputs['setupVersion.ReleaseTitle'] ]
-
-      steps:
-      - task: DownloadBuildArtifacts@0
-        displayName : 'Download Extension wheel from Build Artifacts'
-        inputs:
-          buildType: 'current'
-          downloadType: 'single'
-          artifactName: 'azure-iot'
-          downloadPath: '$(System.ArtifactsDirectory)/extension'
-
-      - task: PowerShell@2
-        displayName: 'Calculate sha for downloaded extension'
-        inputs:
-          targetType: 'inline'
-          script: |
-            $extensions = Get-ChildItem -Filter "*.whl" -Recurse | Select-Object FullName
-            Foreach ($extension in $extensions)
-            {
-                Write-Host "calculating sha256 for " $extension.FullName
-                (Get-Filehash -Path $extension.Fullname -Algorithm SHA256).Hash.ToLower()
-            }
-            Write-Host "done"
-          workingDirectory: '$(System.ArtifactsDirectory)/extension'
-
-      - task: GitHubRelease@1
-        inputs:
-          gitHubConnection: $(GithubReleaseConnection)
-          repositoryName: $(Build.Repository.Name)
-          action: 'create'
-          target: '$(Build.SourceVersion)'
-          tagSource: userSpecifiedTag
-          tag: 'v$(CLIVersion)'
-          title: $(ReleaseTitle)
-          assets: '$(System.ArtifactsDirectory)/extension/**/*.whl'
-          assetUploadMode: 'replace'
-          isDraft: true
-          isPreRelease: false
-          addChangeLog: false
+          architecture: ${{ parameters.architecture }}

.azure-devops/templates/run-tests-parallel.yml

@@ -39,7 +39,6 @@ parameters:
 - name: parallel_execution_dirs
   type: object
   default:
-  - 'azext_iot/tests/central'
   - 'azext_iot/tests/digitaltwins'
   - 'azext_iot/tests/deviceupdate'
 
@@ -81,6 +80,11 @@ steps:
           inlineScript: |
             export COVERAGE_FILE=.coverage.${{ parameters.name }}
             pytest -vv ${{ parameters.path }} -k "_int.py" --dist=loadfile -n ${{ parameters.num_threads }} --reruns ${{ parameters.num_reruns }} --reruns-delay ${{ parameters.reruns_delay }} --cov=azext_iot --cov-config .coveragerc --junitxml=junit/test-iotext-int.xml --durations=0
+        # remove parallelization from central tests
+        ${{ if contains(parameters.path, 'azext_iot/tests/central') }}:
+          inlineScript: |
+            export COVERAGE_FILE=.coverage.${{ parameters.name }}
+            pytest -vv ${{ parameters.path }} -k "_int.py" --cov=azext_iot --cov-config .coveragerc --junitxml=junit/test-iotext-int.xml
         ${{ if contains(parameters.path, 'azext_iot/tests/iothub') }}:
           inlineScript: |
             export COVERAGE_FILE=.coverage.${{ parameters.name }}

.azure-devops/templates/trigger-tests.yml

@@ -92,7 +92,7 @@ jobs:
       serviceConnection: $(AzureServiceConnection)
 
 - job: 'testHub_job_1'
-  timeoutInMinutes: 105
+  timeoutInMinutes: 120
   displayName: 'Test IoT Hub - certificate, config, core, jobs, state'
   condition: and(succeeded(), eq('${{ parameters.testHub }}', true))
   strategy:
@@ -109,7 +109,7 @@ jobs:
       serviceConnection: $(AzureServiceConnection)
 
 - job: 'testHub_job_2'
-  timeoutInMinutes: 90
+  timeoutInMinutes: 120
   condition: and(succeeded(), eq('${{ parameters.testHub }}', true))
   displayName: 'Test IoT Hub - devices, message endpoints, messaging, and modules'
   strategy:
@@ -126,7 +126,7 @@ jobs:
 
 - job: 'testADU'
   timeoutInMinutes: 200
-  displayName: 'Test Azure Device Update'
+  displayName: 'Test ADU'
   condition: and(succeeded(), eq('${{ parameters.testADU }}', true))
   strategy:
     matrix: $[ variables['pythonVersions'] ]
@@ -139,7 +139,7 @@ jobs:
       azureCLIVersion: ${{ parameters.azureCLIVersion }}
       pythonVersion: $(python)
       num_reruns: 0
-      serviceConnection: $(AzureServiceConnectionAlt)
+      serviceConnection: $(AzureServiceConnection)
 
 - job: 'unitTests'
   displayName: 'Unit tests and code coverage'

.github/workflows/release_workflow.yml

@@ -34,6 +34,10 @@ jobs:
     uses: ./.github/workflows/azdev_linter.yml
     with:
       continue-on-error: ${{ github.event.inputs.continue-on-error == 'true' }}
+  int_test:
+    needs: [build]
+    uses: ./.github/workflows/trigger_ado_int_tests.yml
+    secrets: inherit
   approval:
     needs: [security, build, unit-test, azdev_linter]
     # only needed if (release || wheel) - conditionals allow previous jobs to be skipped and still run

.github/workflows/trigger_ado_int_tests.yml

@@ -0,0 +1,191 @@
+name: Run Azure DevOps Integration Tests
+on:
+  workflow_call:
+    inputs:
+      testCentral:
+        description: Run Central tests
+        type: boolean
+        default: true
+      testADT:
+        description: Run ADT tests
+        type: boolean
+        default: true
+      testDPS:
+        description: Run DPS tests
+        type: boolean
+        default: true
+      testHub:
+        description: Run Hub tests
+        type: boolean
+        default: true
+      testADU:
+        description: Run ADU tests
+        type: boolean
+        default: true
+      python_versions:
+        description: "Python versions to test"
+        type: string
+        default: >
+          "{
+              Python38: { python: '3.8' },
+              Python39: { python: '3.9' },
+              Python310: { python: '3.10' }
+          }"
+    secrets:
+      ADO_PAT:
+        required: true
+  workflow_dispatch:
+    inputs:
+      testCentral:
+        description: Run Central tests
+        type: boolean
+        default: true
+      testADT:
+        description: Run ADT tests
+        type: boolean
+        default: true
+      testDPS:
+        description: Run DPS tests
+        type: boolean
+        default: true
+      testHub:
+        description: Run Hub tests
+        type: boolean
+        default: true
+      testADU:
+        description: Run ADU tests
+        type: boolean
+        default: true
+      python_versions:
+        description: "Python versions to test"
+        required: true
+        type: string
+        default: >
+            "{
+                Python38: { python: '3.8' },
+                Python39: { python: '3.9' },
+                Python310: { python: '3.10' }
+            }"
+env:
+  DEFAULT_TITLE: "Azure DevOps Pipeline"
+  AZURE_DEVOPS_EXT_PAT: ${{ secrets.ADO_PAT }}
+  DEFINITION_ID: 147
+  ORG: azureiotdevxp
+  PROJECT: aziotcli
+jobs:
+  call-ado-pipeline:
+    runs-on: ubuntu-latest
+    name: Run integration tests
+    outputs:
+      runId: ${{ steps.start.outputs.runId }}
+      portalUrl: ${{ steps.start.outputs.portalUrl }}
+      url: ${{ steps.start.outputs.url }}
+    steps:
+      - shell: bash
+        id: start
+        name: Trigger the pipeline
+        run: |-
+          set -ex
+          if [ -z "$ORG" ]; then
+            echo "::error::Organization is required"
+            exit 1
+          fi
+          if [ -z "$PROJECT" ]; then
+            echo "::error::Project is required"
+            exit 1
+          fi
+
+          cmdArgs=(--org "https://dev.azure.com/$ORG" --project "$PROJECT" --branch "${{ github.ref }}")
+
+          if [ -n "$DEFINITION_ID" ]; then
+            cmdArgs+=(--id "$DEFINITION_ID")
+          else
+            echo "::error::Pipeline definition ID must be provided"
+            exit 1
+          fi
+
+          python_versions=${{inputs.python_versions}}
+          python_version_strings=$(echo $python_versions | jq -R .)
+          cmdArgs+=("--parameters")
+          cmdArgs+=("pythonVersionsTestingMatrix=$python_version_strings")
+          cmdArgs+=("testCentral=${{inputs.testCentral}}")
+          cmdArgs+=("testADT=${{inputs.testADT}}")
+          cmdArgs+=("testDPS=${{inputs.testDPS}}")
+          cmdArgs+=("testHub=${{inputs.testHub}}")
+          cmdArgs+=("testADU=${{inputs.testADU}}")
+          echo "Running: az pipelines run ${cmdArgs[@]}"
+          res=$(az pipelines run "${cmdArgs[@]}")
+
+          if [ -z "$res" ]; then
+            echo "::error::Failed to trigger the pipeline"
+          fi
+          runId=$(echo $res | jq -r '.id')
+          portalUrl="https://dev.azure.com/$ORG/$PROJECT/_build/results?buildId=$runId&view=results"
+          url=$(echo $res | jq -r '.url')
+          echo "runId=$runId" >> $GITHUB_OUTPUT
+          echo "portalUrl=$portalUrl" >> $GITHUB_OUTPUT
+          echo "url=$url" >> $GITHUB_OUTPUT
+      - name: Summary
+        id: summary
+        run: |-
+          echo "# ${PIPELINE:-$DEFAULT_TITLE} Started" >> $GITHUB_STEP_SUMMARY
+          echo "## Azure DevOps Pipeline" >> $GITHUB_STEP_SUMMARY
+          echo "You can follow the progress [here](${{ steps.start.outputs.portalUrl }})" >> $GITHUB_STEP_SUMMARY
+
+  wait-for-completion:
+    runs-on: ubuntu-latest
+    name: Wait for the pipeline to complete
+    needs: [call-ado-pipeline]
+    env:
+      RUN_ID: ${{ needs.call-ado-pipeline.outputs.runId }}
+      PORTAL_URL: ${{ needs.call-ado-pipeline.outputs.portalUrl }}
+    steps:
+      - name: Wait for the pipeline to complete
+        run: |
+          set -e
+          runId="${{ needs.call-ado-pipeline.outputs.runId }}"
+          status="none"
+          result="none"
+          lastLog=1
+          while [[ "$status" =~ (inProgress|notStarted|postponed|none) ]]; do
+            sleep 60
+            res=$(az pipelines runs show --id "$RUN_ID" --org "https://dev.azure.com/$ORG" --project "$PROJECT")
+            status=$(echo "$res" | jq -r '.status')
+            result=$(echo "$res" | jq -r '.result')
+          done
+
+          # if status is not completed, or the result is one of failed, canceled; then fail the job
+          if [[ "$status" != "completed" || "$result" =~ (failed|canceled) ]]; then
+            echo "::error::Pipeline did not complete successfully"
+            echo "# ❌ ${PIPELINE:-$DEFAULT_TITLE} Failed ❌" >> $GITHUB_STEP_SUMMARY
+            echo "The pipeline did not complete successfully. Please check the logs [here](${PORTAL_URL})" >> $GITHUB_STEP_SUMMARY
+            exit 1
+          fi
+
+          echo "# ✅ ${PIPELINE:-$DEFAULT_TITLE} Completed ✅" >> $GITHUB_STEP_SUMMARY
+          echo "The pipeline completed successfully. You can check the logs [here](${PORTAL_URL})" >> $GITHUB_STEP_SUMMARY
+  CancelPipeline:
+    runs-on: ubuntu-latest
+    needs: [call-ado-pipeline, wait-for-completion]
+    if: cancelled()
+    env:
+      RUN_ID: ${{ needs.call-ado-pipeline.outputs.runId }}
+      RUN_URL: ${{ needs.call-ado-pipeline.outputs.url }}
+      PORTAL_URL: ${{ needs.call-ado-pipeline.outputs.portalUrl }}
+    steps:
+      - name: Cancel the pipeline
+        run: |
+          set -ex
+          if [[ -z "$RUN_ID" ]]; then
+            echo "No pipeline to cancel"
+            exit 0
+          fi
+
+          curl \
+            -X PATCH \
+            -u ":$AZURE_DEVOPS_EXT_PAT" \
+            -H "Content-Type: application/json" \
+            -d '{"status": "cancelling"}' \
+            "${RUN_URL}?api-version=7.2-preview.7"
+          echo "# ❌ ${PIPELINE:-$DEFAULT_TITLE} Cancelled ❌" >> $GITHUB_STEP_SUMMARY
+          echo "The pipeline was cancelled. You can check the logs [here](${PORTAL_URL})" >> $GITHUB_STEP_SUMMARY