Measure JIT TP in superpmi-diffs (#68292)

Author: jakobbotsch

eng/pipelines/coreclr/superpmi-diffs.yml

@@ -28,6 +28,16 @@ jobs:
     jobParameters:
       uploadAs: 'pipelineArtifacts'
 
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/coreclr/templates/build-jit-job.yml
+    buildConfig: release
+    platforms:
+    - windows_x64
+    - windows_x86
+    jobParameters:
+      uploadAs: 'pipelineArtifacts'
+
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
     jobTemplate: /eng/pipelines/coreclr/templates/superpmi-diffs-job.yml

eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml

@@ -73,7 +73,7 @@ jobs:
         mkdir $(SpmiAsmdiffsLocation)
       displayName: Create directories
 
-    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_diffs_setup.py -source_directory $(Build.SourcesDirectory) -product_directory $(buildProductRootFolderPath) -arch $(archType)
+    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_diffs_setup.py -source_directory $(Build.SourcesDirectory) -checked_directory $(buildProductRootFolderPath) -release_directory $(releaseProductRootFolderPath) -arch $(archType)
       displayName: ${{ format('SuperPMI diffs setup ({0})', parameters.archType) }}
 
       # Run superpmi-diffs.py script in helix

eng/pipelines/coreclr/templates/superpmi-diffs-job.yml

@@ -4,7 +4,7 @@ parameters:
   osGroup: ''                     # required -- operating system for the job
   osSubgroup: ''                  # optional -- operating system subgroup
   pool: ''
-  timeoutInMinutes: 120           # build timeout
+  timeoutInMinutes: 180           # build timeout
   variables: {}
   helixQueues: ''
   dependOnEvaluatePaths: false
@@ -13,8 +13,8 @@ parameters:
 jobs:
 - template: ${{ parameters.runJobTemplate }}
   parameters:
-    jobName: ${{ format('superpmi_diffs_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
-    displayName: ${{ format('SuperPMI diffs {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
+    jobName: ${{ format('superpmi_diffs_{0}{1}_{2}', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
+    displayName: ${{ format('SuperPMI diffs {0}{1} {2}', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
     pool: ${{ parameters.pool }}
     buildConfig: ${{ parameters.buildConfig }}
     archType: ${{ parameters.archType }}
@@ -24,16 +24,37 @@ jobs:
     timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
     helixQueues: ${{ parameters.helixQueues }}
     dependsOn:
-     - ${{ format('coreclr_jit_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }}
+     - ${{ format('coreclr_jit_build_{0}{1}_{2}_checked', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
+     - ${{ format('coreclr_jit_build_{0}{1}_{2}_release', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
 
-    variables: ${{ parameters.variables }}
+    variables:
+
+    - ${{ each variable in parameters.variables }}:
+      - ${{ if ne(variable.name, '') }}:
+        - name: ${{ variable.name }}
+          value: ${{ variable.value }}
+      - ${{ if ne(variable.group, '') }}:
+        - group: ${{ variable.group }}
+
+    - name: releaseProductRootFolderPath
+      value: '$(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).Release'
+    - name: releaseProductArtifactName
+      value: 'CoreCLRProduct_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_$(archType)_release'
 
     steps:
 
-    # Download jit builds
+    # Download jit checked builds
     - template: /eng/pipelines/common/download-artifact-step.yml
       parameters:
         unpackFolder: $(buildProductRootFolderPath)
         artifactFileName: '$(buildProductArtifactName)$(archiveExtension)'
         artifactName: '$(buildProductArtifactName)'
-        displayName: 'JIT product build'
+        displayName: 'JIT checked build'
+
+    #Download jit release builds
+    - template: /eng/pipelines/common/download-artifact-step.yml
+      parameters:
+        unpackFolder: $(releaseProductRootFolderPath)
+        artifactFileName: '$(releaseProductArtifactName)$(archiveExtension)'
+        artifactName: '$(releaseProductArtifactName)'
+        displayName: 'JIT release build'

src/coreclr/scripts/jitutil.py

@@ -605,7 +605,7 @@ def download_with_progress_urlretrieve(uri, target_location, fail_if_not_found=T
             if try_num == num_tries - 1:
                 raise ex
 
-            if ex is urllib.error.HTTPError and ex == 404:
+            if isinstance(ex, urllib.error.HTTPError) and ex.code == 404:
                 if fail_if_not_found:
                     logging.error("HTTP 404 error")
                     raise ex

src/coreclr/scripts/superpmi-diffs.proj

@@ -31,7 +31,7 @@
     <!-- Workaround until https://github.com/dotnet/arcade/pull/6179 is not available -->
     <HelixResultsDestinationDir>$(BUILD_SOURCESDIRECTORY)\artifacts\helixresults</HelixResultsDestinationDir>
     <WorkItemCommand>$(Python) $(ProductDirectory)\superpmi_diffs.py -base_jit_directory $(ProductDirectory)\base -diff_jit_directory $(ProductDirectory)\diff -log_directory $(SuperpmiLogsLocation)</WorkItemCommand>
-    <WorkItemTimeout>1:00</WorkItemTimeout>
+    <WorkItemTimeout>2:00</WorkItemTimeout>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -68,7 +68,7 @@
     <HelixWorkItem Include="@(SPMI_Partition)">
       <Command>$(WorkItemCommand) -arch %(HelixWorkItem.Architecture) -platform %(HelixWorkItem.Platform)</Command>
       <Timeout>$(WorkItemTimeout)</Timeout>
-      <DownloadFilesFromResults>superpmi_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_download_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_diff_summary_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).md;Asmdiffs_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).zip</DownloadFilesFromResults>
+      <DownloadFilesFromResults>superpmi_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_download_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).log;superpmi_diff_summary_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).md;superpmi_tpdiff_summary_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).md;Asmdiffs_%(HelixWorkItem.Platform)_%(HelixWorkItem.Architecture).zip</DownloadFilesFromResults>
     </HelixWorkItem>
   </ItemGroup>
   </Project>