[Eng] Added necessary tools/pipelines update to migrate main to future

.azure-pipelines/PipelineSteps/README.md

@@ -0,0 +1,7 @@
+# Pipeline Steps
+
+This directory contains the PowerShell scripts refactored from ".azure-pipelines/util".
+
+## Overview
+
+The pipeline steps are refactored from yml based templates from ".azure-pipelines/util" to PowerShell scripts. This is designed for converting some Azure Devops pipelines to github action.

.azure-pipelines/PipelineSteps/analyze-steps.ps1

@@ -0,0 +1,44 @@
+# Copyright Microsoft Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ----------------------------------------------------------------------------------
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [string]$RepoRoot,
+    [string]$Configuration = 'Debug',
+    [string]$PowerShellPlatform
+)
+
+# Install PowerShell dependencies
+Write-Host -ForegroundColor Green "-------------------- Start installing PowerShell dependencies ... --------------------"
+Install-Module "platyPS", "PSScriptAnalyzer" -Repository PSGallery -Force -Confirm:$false -Scope CurrentUser
+Write-Host -ForegroundColor DarkGreen "-------------------- End installing PowerShell dependencies ... --------------------`n`n`n`n`n"
+
+# Install latest Az modules
+Write-Host -ForegroundColor Green "-------------------- Start installing latest Az modules ... --------------------"
+New-Item -ItemType Directory -Path "Az-Cmdlets-latest"
+Invoke-WebRequest -Uri "https://azpspackage.blob.core.windows.net/release/Az-Cmdlets-latest.tar.gz" -OutFile "Az-Cmdlets-latest/Az-Cmdlets-latest.tar.gz" -MaximumRetryCount 2 -RetryIntervalSec 1
+tar -xvzf "Az-Cmdlets-latest/Az-Cmdlets-latest.tar.gz" -C "Az-Cmdlets-latest"
+& Az-Cmdlets-latest/InstallModule.ps1
+Write-Host -ForegroundColor DarkGreen "-------------------- End installing latest Az modules ... --------------------`n`n`n`n`n"
+
+# Generate help
+Write-Host -ForegroundColor Green "-------------------- Start generating help ... --------------------"
+if ($PowerShellPlatform) {
+    $Env:PowerShellPlatform = $PowerShellPlatform
+}
+$buildProjPath = Join-Path $RepoRoot 'build.proj'
+dotnet msbuild $buildProjPath /t:GenerateHelp "/p:Configuration=$Configuration"
+Write-Host -ForegroundColor DarkGreen "-------------------- End generating help ... --------------------`n`n`n`n`n"
+
+# Static Analysis
+Write-Host -ForegroundColor Green "-------------------- Start static analysis ... --------------------"
+dotnet msbuild $buildProjPath /t:StaticAnalysis "/p:Configuration=$Configuration"
+Write-Host -ForegroundColor DarkGreen "-------------------- End static analysis ... --------------------`n`n`n`n`n"

.azure-pipelines/PipelineSteps/build-steps.ps1

@@ -0,0 +1,118 @@
+# ----------------------------------------------------------------------------------
+#
+# Copyright Microsoft Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ----------------------------------------------------------------------------------
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [string]$RepoRoot,
+    [string]$Configuration = 'Debug',
+    [string]$PowerShellPlatform,
+    [string]$FilesChangedOutputPath,
+    [boolean]$ForceRegenerate = $false,
+    [string]$SubTasksFilePath,
+    [boolean]$IsSecurityCheck,
+    [string]$BuildReason,
+    [string]$Trigger
+)
+
+# filter changed files
+Write-Host -ForegroundColor Green "-------------------- Start filtering changed files ... --------------------"
+$buildProjPath = Join-Path $RepoRoot 'build.proj'
+if ($PowerShellPlatform) {
+    $Env:PowerShellPlatform = $PowerShellPlatform
+}
+dotnet msbuild $buildProjPath /t:FilterBuild "/p:FilesChangedOutputPath=$FilesChangedOutputPath;SubTasksFilePath=$SubTasksFilePath;IsSecurityCheck=$IsSecurityCheck"
+Write-Host -ForegroundColor DarkGreen "-------------------- End filtering changed files ... --------------------`n`n`n`n`n"
+
+# Set subtasks
+if (Test-Path $SubTasksFilePath) {
+    Write-Host -ForegroundColor Green "-------------------- Start setting subtasks ... --------------------"
+    Get-Content $SubTasksFilePath | ForEach-Object {
+      if ($_ && 'Predictor' -eq $_) {
+        $subTaskPredictor = $true
+      } elseif ($_ && 'Installer' -eq $_) {
+        Write-Host "##vso[task.setvariable variable=SubTaskInstaller]true"
+        $subTaskInstaller = $true
+      } elseif ($_ && 'all' -eq $_) {
+        $subTaskAll = $true
+      }
+    }
+    Write-Host -ForegroundColor DarkGreen "-------------------- End setting subtasks ... --------------------`n`n`n`n`n"
+}
+
+# Analyze changed files
+Write-Host -ForegroundColor Green "-------------------- Start Analyzing changed files ... --------------------"
+Write-Host "##1. Check generate-info.json for autorest generated modules" -ForegroundColor Magenta
+$noGenerateInfo = @()
+$modules = Get-Content $FilesChangedOutputPath -OutVariable paths | Foreach-Object { if($_ -match "^src.*\.Autorest"){ Write-Output $Matches[0] } } | Select-Object -Unique
+foreach ($module in $modules) {
+$hasGenerateInfo = $false
+$pattern = "^$module/generate-info.json"
+foreach ($path in $paths) {
+    if ($path -match $pattern) {
+    $hasGenerateInfo = $true
+    }
+}
+if (-not $hasGenerateInfo) {
+    $noGenerateInfo += $module
+}
+}
+if ($noGenerateInfo -and $noGenerateInfo.Count -gt 0) {
+$noGenerateInfo | Foreach-Object { Write-Warning "No generate-info.json detected for $_." }
+}
+
+# this might not be true, for example when release branch or preview branch merge back to main branch, it's possible changes present in both /src and /generated
+Write-Host "##2. PR should contain changes from either /src or /generated only" -ForegroundColor Magenta
+$content = Get-Content $FilesChangedOutputPath
+$srcFiles = $content | Where-Object { $_ -match "^src.*" }
+$generatedFiles = $content | Where-Object { $_ -match "^generated.*" }
+if ($srcFiles -and $srcFiles.Count -gt 0  -and $generatedFiles -and $generatedFiles.Count -gt 0) {
+Write-Warning "PR should contain changes from either /src or /generated only."
+}
+Write-Host -ForegroundColor DarkGreen "-------------------- End Analyzing changed files ... --------------------`n`n`n`n`n"
+
+# Check ignored files
+Write-Host -ForegroundColor Green "-------------------- Start checking ignored files ... --------------------"
+$checkIgnoreFilesScriptPath = Join-Path $RepoRoot 'tools' 'CheckIgnoredFile.ps1'
+& $checkIgnoreFilesScriptPath
+Write-Host -ForegroundColor DarkGreen "-------------------- End checking ignored files ... --------------------`n`n`n`n`n"
+
+# Build
+Write-Host -ForegroundColor Green "-------------------- Start building modules ... --------------------"
+$buildScriptPath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildModules.ps1'
+if ('Manual' -eq $BuildReason -and $ForceRegenerate) {
+& $buildScriptPath -CIPlan -RepoRoot $RepoRoot -Configuration $Configuration -ForceRegenerate -InvokedByPipeline
+} else {
+& $buildScriptPath -CIPlan -RepoRoot $RepoRoot -Configuration $Configuration -InvokedByPipeline
+}
+Write-Host -ForegroundColor DarkGreen "-------------------- End building modules ... --------------------`n`n`n`n`n"
+
+# Write pipeline result
+Write-Host -ForegroundColor Green "-------------------- Start writing pipeline result ... --------------------"
+$pipelineScript = Join-Path $RepoRoot 'tools' 'ExecuteCIStep.ps1'
+$repoArtifact = Join-Path $RepoRoot 'artifacts'
+& $pipelineScript -Build -TriggerType $BuildReason -Trigger $Trigger -RepoArtifacts $repoArtifact -Configuration $Configuration
+Write-Host -ForegroundColor DarkGreen "-------------------- End writing pipeline result ... --------------------`n`n`n`n`n"
+
+# Build Az.Tools.Predictor
+if ($subTaskPredictor -or $subTaskAll) {
+    Write-Host -ForegroundColor Green "-------------------- Start building Az.Tools.Predictor ... --------------------"
+    dotnet msbuild $buildProjPath /t:AzToolsPredictor
+    Write-Host -ForegroundColor DarkGreen "-------------------- End building Az.Tools.Predictor ... --------------------`n`n`n`n`n"
+}
+
+# Build Az.Tools.Installer
+if ($subTaskInstaller -or $subTaskAll) {
+    Write-Host -ForegroundColor Green "-------------------- Start building Az.Tools.Installer ... --------------------"
+    dotnet msbuild $buildProjPath /t:AzToolsInstaller
+    Write-Host -ForegroundColor DarkGreen "-------------------- End building Az.Tools.Installer ... --------------------`n`n`n`n`n"
+}

.azure-pipelines/PipelineSteps/test-steps.ps1

@@ -0,0 +1,77 @@
+# Copyright Microsoft Corporation
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ----------------------------------------------------------------------------------
+[CmdletBinding(DefaultParameterSetName="AllSet")]
+param (
+    [string]$RepoRoot,
+    [string]$PowerShellPlatform,
+    [string]$TestFramework,
+    [string]$Configuration = 'Debug'
+)
+
+# Remove pre-installed Az modules
+Write-Host -ForegroundColor Green "-------------------- Start removing pre-installed Az modules ... --------------------"
+$commonUtilityScriptPath = Join-Path $RepoRoot "tools" "TestFx" "Utilities" "CommonUtility.ps1"
+& $commonUtilityScriptPath
+Write-Host -ForegroundColor DarkGreen "-------------------- End removing pre-installed Az modules ... --------------------`n`n`n`n`n"
+
+# Test
+Write-Host -ForegroundColor Green "-------------------- Start testing ... --------------------"
+if ($PowerShellPlatform) {
+    $Env:PowerShellPlatform = $PowerShellPlatform
+}
+$preference = $ErrorActionPreference
+$ErrorActionPreference = 'Continue'
+$buildProjPath = Join-Path $RepoRoot 'build.proj'
+dotnet msbuild $buildProjPath /t:Test "/p:Configuration=$Configuration;TestFramework=$TestFramework"
+Write-Host -ForegroundColor DarkGreen "-------------------- End testing ... --------------------`n`n`n`n`n"
+
+# Test AutoGen Modules With PowerShell Core
+Write-Host -ForegroundColor Green "-------------------- Start testing AutoGen modules with PowerShell Core ... --------------------"
+$executeCIStepScriptPath = Join-Path $RepoRoot "tools" "ExecuteCIStep.ps1"
+$currentPath = $PWD
+$debugFolderPath = Join-Path $RepoRoot "artifacts" "Debug"
+Set-Location $debugFolderPath
+
+Install-Module -Name Pester -Repository PSGallery -RequiredVersion 4.10.1 -Force
+if ($IsWindows) { $sp = ";" } else { $sp = ":" }
+$env:PSModulePath = $env:PSModulePath + $sp + (pwd).Path
+Get-ChildItem -File -Recurse test-module.ps1 | ForEach-Object {
+Write-Host $_.Directory.FullName
+$repoArtifact = Join-Path $RepoRoot 'artifacts'
+& $executeCIStepScriptPath -TestAutorest -AutorestDirectory $_.Directory.FullName -RepoArtifacts $repoArtifact
+}
+
+$ErrorActionPreference = $preference
+Set-Location $currentPath
+Write-Host -ForegroundColor DarkGreen "-------------------- End testing AutoGen modules with PowerShell Core ... --------------------`n`n`n`n`n"
+
+# Analyze test coverage
+Write-Host -ForegroundColor Green "-------------------- Start analyzing test coverage ... --------------------"
+$validateTestCoverageScriptPath = Join-Path $RepoRoot 'tools' 'TestFx' 'Coverage' 'ValidateTestCoverage.ps1'
+& $validateTestCoverageScriptPath
+Write-Host -ForegroundColor DarkGreen "-------------------- End analyzing test coverage ... --------------------`n`n`n`n`n"
+
+# Check test status
+Write-Host -ForegroundColor Green "-------------------- Start checking test status ... --------------------"
+$currentPath = $PWD
+$pipelineResultPath = Join-Path $RepoRoot "artifacts" "PipelineResult"
+Set-Location $pipelineResultPath
+
+$PipelineResult = Get-Content PipelineResult.json | ConvertFrom-Json
+$FailedModuleList = $PipelineResult.test.Details[0].Modules | Where-Object { $_.Status -eq "Failed" } | ForEach-Object { $_.Module }
+if ($FailedModuleList.Length -ne 0)
+{
+    throw "test fails in module: $FailedModuleList"
+}
+
+Set-Location $currentPath
+Write-Host -ForegroundColor DarkGreen "-------------------- End checking test status ... --------------------`n`n`n`n`n"