Remove Python 3.8 + Standardize on azsdk-pool on affected templates (Azure#40499)

* standardize on new pools + directly update 3.8 to 3.9 and remove 3.8 from livetest matrices

Co-authored-by: Yuchao Yan <[email protected]>

Author: scbedd

eng/pipelines/autorest_checks.yml

@@ -25,8 +25,8 @@ jobs:
     displayName: 'Run AutoRest'
 
     pool:
-      name: azsdk-pool-mms-ubuntu-2004-general
-      vmImage: 'ubuntu-20.04'
+      name: azsdk-pool
+      vmImage: 'ubuntu-22.04'
 
     steps:
       - task: NodeTool@0

eng/pipelines/templates/jobs/smoke.tests.yml

@@ -13,8 +13,8 @@ jobs:
     - job: smoke_test_eligibility
       displayName: Check Smoke Test Eligibility
       pool:
-        name: azsdk-pool-mms-ubuntu-2004-general
-        image: azsdk-pool-mms-ubuntu-2004-1espt
+        name: $(LINUXPOOL)
+        image: $(LINUXVMIMAGE)
         os: linux
       steps:
         - ${{ if and(ne(variables['Skip.Release'], 'true'), ne(parameters.Artifact.skipPublishPackage, 'true')) }}:
@@ -41,23 +41,20 @@ jobs:
       condition: and(succeeded(), eq(dependencies.smoke_test_eligibility.outputs['output_eligibility.RunSmokeTests'], true))
     strategy:
       matrix:
-        Python_38_Linux (AzureCloud):
-          PythonVersion: '3.8'
-          Pool: "azsdk-pool-mms-ubuntu-2004-general"
-          OSVmImage: azsdk-pool-mms-ubuntu-2004-1espt
+        Python_310_Linux (AzureCloud):
+          PythonVersion: '3.10'
         ${{ if eq(parameters.Daily, true) }}:
-          Python_37_Linux (AzureCloud):
-            PythonVersion: '3.7'
-            Pool: azsdk-pool-mms-ubuntu-2004-general
-            OSVmImage: azsdk-pool-mms-ubuntu-2004-1espt
+          Python_39_Linux (AzureCloud):
+            PythonVersion: '3.9'
 
     pool:
-      name: $(Pool)
-      image: $(OSVmImage)
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
       os: linux
 
     variables:
       - template: /eng/pipelines/templates/variables/globals.yml
+      - template: /eng/pipelines/templates/variables/image.yml
       - name: requirements
         ${{ if eq(parameters.Daily, true) }}:
           value: $(Build.SourcesDirectory)/common/smoketest/requirements.txt
@@ -79,21 +76,18 @@ jobs:
         condition: and(succeeded(), eq(dependencies.smoke_test_eligibility.outputs['output_eligibility.RunSmokeTests'], true))
       strategy:
         matrix:
-            Python_37_Windows (AzureCloud):
-              PythonVersion: '3.7'
-              Pool: azsdk-pool-mms-win-2022-general
-              OSVmImage: azsdk-pool-mms-win-2022-1espt
-            Python_38_Windows (AzureCloud):
-              PythonVersion: '3.8'
-              Pool: azsdk-pool-mms-win-2022-general
-              OSVmImage: azsdk-pool-mms-win-2022-1espt
+            Python_310_Windows (AzureCloud):
+              PythonVersion: '3.10'
+            Python_39_Windows (AzureCloud):
+              PythonVersion: '3.9'
 
       pool:
-        name: $(Pool)
-        image: $(OSVmImage)
+        name: $(WINDOWSPOOL)
+        image: $(WINDOWSVMIMAGE)
         os: windows
       variables:
         - template: /eng/pipelines/templates/variables/globals.yml
+        - template: /eng/pipelines/templates/variables/image.yml
         - name: requirements
           ${{ if eq(parameters.Daily, true) }}:
             value: $(Build.SourcesDirectory)/common/smoketest/requirements.txt
@@ -113,22 +107,19 @@ jobs:
         condition: and(succeeded(), eq(dependencies.smoke_test_eligibility.outputs['output_eligibility.RunSmokeTests'], true))
       strategy:
         matrix:
-            Python_37_Mac (AzureCloud):
-              PythonVersion: '3.7'
-              Pool: Azure Pipelines
-              OSVmImage: macos-latest
-            Python_38_Mac (AzureCloud):
-              PythonVersion: '3.8'
-              Pool: Azure Pipelines
-              OSVmImage: macos-latest
+            Python_310_Mac (AzureCloud):
+              PythonVersion: '3.10'
+            Python_39_Mac (AzureCloud):
+              PythonVersion: '3.9'
 
       pool:
-        name: $(Pool)
-        vmImage: $(OSVmImage)
+        name: $(MACPOOL)
+        vmImage: $(MACVMIMAGE)
         os: macOS
 
       variables:
         - template: /eng/pipelines/templates/variables/globals.yml
+        - template: /eng/pipelines/templates/variables/image.yml
         - name: requirements
           ${{ if eq(parameters.Daily, true) }}:
             value: $(Build.SourcesDirectory)/common/smoketest/requirements.txt

eng/pipelines/templates/jobs/update_pr.yml

@@ -10,14 +10,14 @@ variables:
   skipComponentGovernanceDetection: true
 
 pool:
-  name: azsdk-pool-mms-ubuntu-2204-general
-  vmImage: 'ubuntu-22.04'
+  name: $(LINUXPOOL)
+  vmImage: $(LINUXVMIMAGE)
 
 steps:
 - task: UsePythonVersion@0
-  displayName: 'Use Python 3.8'
+  displayName: 'Use Python 3.9'
   inputs:
-    versionSpec: 3.8
+    versionSpec: 3.9
 
 - script: |
     python3 -m pip install -e $(Build.SourcesDirectory)/tools/azure-sdk-tools[ghtools]

eng/pipelines/templates/stages/archetype-python-release.yml

@@ -16,6 +16,8 @@ stages:
       - stage:
         displayName: 'Release: ${{artifact.name}}'
         dependsOn: ${{parameters.DependsOn}}
+        variables:
+          - template: /eng/pipelines/templates/variables/image.yml
         condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr'))
         jobs:
           - job: TagRepository

eng/pipelines/templates/stages/platform-matrix.json

@@ -9,7 +9,7 @@
       "ubuntu-20.04": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" },
       "windows-2022": { "OSVmImage": "env:WINDOWSVMIMAGE", "Pool": "env:WINDOWSPOOL" }
     },
-    "PythonVersion": [ "3.8", "3.10", "3.12" ],
+    "PythonVersion": [ "3.10", "3.12" ],
     "CoverageArg": "--disablecov",
     "TestSamples": "false"
   },

eng/pipelines/templates/stages/python-analyze-weekly.yml

@@ -19,20 +19,16 @@ stages:
       - job: 'Analyze'
         timeoutInMinutes: 90
 
-        variables:
-          - name: PythonVersion
-            value: '3.8'
-
         pool:
           name: $(LINUXPOOL)
           image: $(LINUXVMIMAGE)
           os: linux
 
         steps:
           - task: UsePythonVersion@0
-            displayName: 'Use Python 3.8'
+            displayName: 'Use Python 3.9'
             inputs:
-              versionSpec: '3.8'
+              versionSpec: '3.9'
           - script: |
               python -m pip install -r eng/ci_tools.txt
             displayName: 'Prep Environment'

eng/pipelines/templates/steps/run_black.yml

@@ -5,9 +5,9 @@ parameters:
 
 steps:
   - task: UsePythonVersion@0
-    displayName: 'Use Python 3.8'
+    displayName: 'Use Python 3.9'
     inputs:
-     versionSpec: '3.8'
+     versionSpec: '3.9'
     condition: succeededOrFailed()
 
   - script: |

eng/pipelines/templates/steps/run_mypy.yml

@@ -8,9 +8,9 @@ parameters:
 # Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml.
 steps:
   - task: UsePythonVersion@0
-    displayName: 'Use Python 3.8'
+    displayName: 'Use Python 3.9'
     inputs:
-     versionSpec: '3.8'
+     versionSpec: '3.9'
     condition: and(succeededOrFailed(), ne(variables['Skip.MyPy'],'true'))
 
   - script: |

eng/pipelines/templates/steps/run_pylint.yml

@@ -7,9 +7,9 @@ parameters:
 # Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml.
 steps:
   - task: UsePythonVersion@0
-    displayName: 'Use Python 3.8'
+    displayName: 'Use Python 3.9'
     inputs:
-     versionSpec: '3.8'
+     versionSpec: '3.9'
     condition: and(succeededOrFailed(), ne(variables['Skip.Pylint'],'true'))
 
   - script: |

eng/pipelines/templates/steps/run_pyright.yml

@@ -7,9 +7,9 @@ parameters:
 # Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml.
 steps:
   - task: UsePythonVersion@0
-    displayName: 'Use Python 3.8'
+    displayName: 'Use Python 3.9'
     inputs:
-     versionSpec: '3.8'
+     versionSpec: '3.9'
     condition: and(succeededOrFailed(), or(ne(variables['Skip.Pyright'],'true'), ne(variables['Skip.Verifytypes'],'true')))
 
   - script: |

eng/pipelines/templates/steps/smoke-test-steps.yml

@@ -9,10 +9,6 @@ parameters:
     default: "not-specified"
 
 steps:
-  - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
-    parameters:
-      AgentImage: $(OSVmImage)
-
   - task: UsePythonVersion@0
     displayName: "Use Python $(PythonVersion)"
     inputs:

pyrightconfig.json

@@ -1,5 +1,5 @@
 {
   "reportTypeCommentUsage": true,
   "reportMissingImports": false,
-  "pythonVersion": "3.8"
+  "pythonVersion": "3.9"
 }

sdk/communication/azure-communication-phonenumbers/phonenumbers-livetest-matrix.json

@@ -6,7 +6,7 @@
   },
   "matrix": {
     "Agent": {
-      "ubuntu-20.04": {
+      "ubuntu-22.04": {
         "OSVmImage": "env:LINUXVMIMAGE",
         "Pool": "env:LINUXPOOL",
         "COMMUNICATION_SKIP_CAPABILITIES_LIVE_TEST": "true"
@@ -24,7 +24,7 @@
         "COMMUNICATION_SKIP_CAPABILITIES_LIVE_TEST": "false"
       }
     },
-    "PythonVersion": ["pypy3.9", "3.8", "3.10", "3.11"],
+    "PythonVersion": ["pypy3.9", "3.10", "3.11"],
     "CoverageArg": "--disablecov",
     "TestSamples": "false"
   },

sdk/core/azure-core/perf.yml

@@ -1,6 +1,6 @@
 parameters:
 - name: LanguageVersion
-  displayName: LanguageVersion (3.8, 3.9, 3.10, 3.11)
+  displayName: LanguageVersion (3.9, 3.10, 3.11)
   type: string
   default: '3.11'
 - name: PackageVersions

sdk/core/corehttp/perf.yml

@@ -1,6 +1,6 @@
 parameters:
 - name: LanguageVersion
-  displayName: LanguageVersion (3.8, 3.9, 3.10, 3.11, 3.12)
+  displayName: LanguageVersion (3.9, 3.10, 3.11, 3.12)
   type: string
   default: '3.11'
 - name: PackageVersions

sdk/cosmos/cosmos-emulator-matrix.json

@@ -9,15 +9,6 @@
         "windows-2022": { "OSVmImage": "env:WINDOWSVMIMAGE", "Pool": "env:WINDOWSPOOL" }
       },
       "EmulatorConfig": {
-        "Emulator Tests Python 3.8": {
-            "PythonVersion": "3.8",
-            "CoverageArg": "--disablecov",
-            "TestSamples": "false",
-            "ToxTestEnv": "'whl,sdist'",
-            "ACCOUNT_HOST": "https://localhost:8081/",
-            "ACCOUNT_KEY": "C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==",
-            "Skip.Analyze": "true"
-        },
         "Emulator Tests Python 3.9": {
             "PythonVersion": "3.9",
             "CoverageArg": "--disablecov",

sdk/cosmos/live-platform-matrix.json

@@ -63,14 +63,6 @@
     },
     {
       "WindowsConfig": {
-        "Windows2022_38_long": {
-          "OSVmImage": "env:WINDOWSVMIMAGE",
-          "Pool": "env:WINDOWSPOOL",
-          "PythonVersion": "3.8",
-          "CoverageArg": "--disablecov",
-          "TestSamples": "false",
-          "TestMarkArgument": "cosmosLong"
-        },
         "Windows2022_310_long": {
           "OSVmImage": "env:WINDOWSVMIMAGE",
           "Pool": "env:WINDOWSPOOL",

sdk/eventhub/perf.yml

@@ -1,6 +1,6 @@
 parameters:
 - name: LanguageVersion
-  displayName: LanguageVersion (3.7, 3.8, 3.9, 3.10, 3.11)
+  displayName: LanguageVersion (3.9, 3.10, 3.11)
   type: string
   default: '3.11'
 - name: PackageVersions

sdk/identity/platform-matrix.json

@@ -8,10 +8,10 @@
   "include": [
     {
       "Config": {
-        "ubuntu_20.04_3.8_msal": {
+        "ubuntu_20.04_3.9_msal": {
           "OSVmImage": "env:LINUXVMIMAGE",
           "Pool": "env:LINUXPOOL",
-          "PythonVersion": "3.8",
+          "PythonVersion": "3.9",
           "CoverageArg": "--disablecov",
           "InjectedPackages": "git+https://github.com/AzureAD/microsoft-authentication-library-for-python@dev",
           "UnsupportedToxEnvironments": "mindependency,latestdependency"

sdk/keyvault/azure-keyvault-secrets/pyrightconfig.json

@@ -1,6 +1,6 @@
 {
   "reportMissingImports": false,
-  "pythonVersion": "3.8",
+  "pythonVersion": "3.9",
   "exclude": [
     "**/_generated/**"
   ]

sdk/servicebus/perf.yml

@@ -1,6 +1,6 @@
 parameters:
 - name: LanguageVersion
-  displayName: LanguageVersion (3.7, 3.8, 3.9, 3.10, 3.11)
+  displayName: LanguageVersion (3.9, 3.10, 3.11)
   type: string
   default: '3.11'
 - name: PackageVersions

sdk/storage/azure-storage-blob/perf.yml

@@ -1,6 +1,6 @@
 parameters:
 - name: LanguageVersion
-  displayName: LanguageVersion (3.7, 3.8, 3.9, 3.10, 3.11)
+  displayName: LanguageVersion (3.9, 3.10, 3.11)
   type: string
   default: '3.11'
 - name: PackageVersions

sdk/storage/azure-storage-file-datalake/perf.yml

@@ -1,6 +1,6 @@
 parameters:
 - name: LanguageVersion
-  displayName: LanguageVersion (3.7, 3.8, 3.9, 3.10, 3.11)
+  displayName: LanguageVersion (3.9, 3.10, 3.11)
   type: string
   default: '3.11'
 - name: PackageVersions

sdk/storage/azure-storage-file-share/perf.yml

@@ -1,6 +1,6 @@
 parameters:
 - name: LanguageVersion
-  displayName: LanguageVersion (3.7, 3.8, 3.9, 3.10, 3.11)
+  displayName: LanguageVersion (3.9, 3.10, 3.11)
   type: string
   default: '3.11'
 - name: PackageVersions
@@ -14,7 +14,7 @@ parameters:
 - name: Arguments
   displayName: Arguments (regex of arguments to run)
   type: string
-  default: '(10240)|(10485760)|(1073741824)' 
+  default: '(10240)|(10485760)|(1073741824)'
 - name: Iterations
   displayName: Iterations (times to run each test)
   type: number

sdk/storage/platform-matrix-all-versions.json

@@ -9,7 +9,7 @@
           "Ubuntu-20.04": { "OSVmImage": "env:LINUXVMIMAGE", "Pool": "env:LINUXPOOL" },
           "windows-2022": { "OSVmImage": "env:WINDOWSVMIMAGE", "Pool": "env:WINDOWSPOOL" }
         },
-        "PythonVersion": [ "3.8", "3.9", "3.10"],
+        "PythonVersion": [ "3.9", "3.10"],
         "CoverageArg": "--disablecov",
         "TestSamples": "false",
         "AZURE_LIVE_TEST_SERVICE_VERSION": [