Skip to content

Commit 3aa769d

Browse files
Pan-QiPan-Qi
authored
Batch generation (#27534)
1 parent 881baa3 commit 3aa769d

13 files changed

+1327
-0
lines changed

‎.azure-pipelines/PipelineSteps/BatchGeneration/FailedJobTeamsTemplate.html

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<!DOCTYPE html>
2+
<html>
3+
4+
<head>
5+
<meta charset="utf-8">
6+
</head>
7+
8+
<body>
9+
<h1 style="color: #FF0000; font-size: 24px; font-weight: bold;">Pipeline Failure Alert 🚨</h1>
10+
<p style="font-size: 18px;">The pipeline **{{ pipelineName }}** has failed.</p>
11+
<p style="font-size: 16px;">You can review the details at the following links:</p>
12+
<ul style="list-style-type: none; padding-left: 0;">
13+
<li style="font-size: 16px; margin: 10px 0;">
14+
<a href="{{ pipelineUrl }}" style="color: #0078D4; text-decoration: none;">📝 Pipeline Overview</a>
15+
</li>
16+
<li style="font-size: 16px; margin: 10px 0;">
17+
<a href="{{ runUrl }}" style="color: #0078D4; text-decoration: none;">❌ Failed Run Details</a>
18+
</li>
19+
</ul>
20+
<hr style="border: none; height: 1px; background-color: #0078D4; margin: 20px 0;">
21+
<p style="font-size: 16px;">Please check and address the issue as soon as possible.</p>
22+
<p style="font-size: 16px; margin-top: 20px; font-style: italic;">Sincerely,</p>
23+
<p style="font-size: 16px; font-weight: bold;">Your Azure CLI Tools Team</p>
24+
</body>
25+
26+
</html>

‎.azure-pipelines/PipelineSteps/BatchGeneration/analyse-modules.ps1

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
param (
2+
[string]$MatrixKey,
3+
[string]$RepoRoot
4+
)
5+
6+
$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
7+
Import-Module $utilFilePath -Force
8+
$moduleGroup = Get-Targets -RepoRoot $RepoRoot -TargetsOutputFileName "analyzeTargets.json" -MatrixKey $MatrixKey
9+
$RepoArtifacts = Join-Path $RepoRoot 'artifacts'
10+
$StaticAnalysisOutputDirectory = Join-Path $RepoArtifacts 'StaticAnalysisResults'
11+
if (-not (Test-Path -Path $StaticAnalysisOutputDirectory)) {
12+
New-Item -ItemType Directory -Path $StaticAnalysisOutputDirectory
13+
}
14+
$toolsDirectory = Join-Path $RepoRoot 'tools'
15+
16+
$results = @()
17+
foreach ($moduleName in $moduleGroup) {
18+
Write-Host "=============================================================="
19+
Write-Host "Analysing Module: $moduleName"
20+
21+
$startTime = Get-Date
22+
$result = @{
23+
MatrixKey = $MatrixKey
24+
Module = $moduleName
25+
Status = "Success"
26+
DurationSeconds = 0
27+
Error = ""
28+
FailedTasks = @()
29+
}
30+
$Parameters = @{
31+
RepoArtifacts = $RepoArtifacts
32+
StaticAnalysisOutputDirectory = $StaticAnalysisOutputDirectory
33+
Configuration = "Debug"
34+
TargetModule = @($moduleName)
35+
}
36+
$FailedTasks = @()
37+
$ErrorLogPath = "$StaticAnalysisOutputDirectory/error.log"
38+
39+
try {
40+
.("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisBreakingChange @Parameters 2>$ErrorLogPath
41+
If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
42+
{
43+
$FailedTasks += "BreakingChange"
44+
}
45+
.("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisDependency @Parameters 2>>$ErrorLogPath
46+
If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
47+
{
48+
$FailedTasks += "Dependency"
49+
}
50+
.("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisSignature @Parameters 2>>$ErrorLogPath
51+
If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
52+
{
53+
$FailedTasks += "Signature"
54+
}
55+
.("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisHelp @Parameters 2>>$ErrorLogPath
56+
If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
57+
{
58+
$FailedTasks += "Help"
59+
}
60+
.("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisUX @Parameters 2>>$ErrorLogPath
61+
If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
62+
{
63+
$FailedTasks += "UXMetadata"
64+
}
65+
.("$toolsDirectory/ExecuteCIStep.ps1") -StaticAnalysisCmdletDiff @Parameters 2>>$ErrorLogPath
66+
If (($LASTEXITCODE -ne 0) -and ($LASTEXITCODE -ne $null))
67+
{
68+
$FailedTasks += "CmdletDiff"
69+
}
70+
If ($FailedTasks.Length -ne 0)
71+
{
72+
Write-Host "There are failed tasks: $FailedTasks"
73+
$ErrorLog = Get-Content -Path $ErrorLogPath | Join-String -Separator "`n"
74+
Write-Error $ErrorLog
75+
$result.Status = "Failed"
76+
$result.Error = "Failed tasks: $($FailedTasks -join ', ')"
77+
$result.FailedTasks = $FailedTasks
78+
}
79+
} catch {
80+
Write-Warning "Failed to analyse module: $moduleName"
81+
Write-Warning "Error message: $($_.Exception.Message)"
82+
$result.Status = "Failed"
83+
$result.Error = $_.Exception.Message
84+
} finally {
85+
$endTine = Get-Date
86+
$result.DurationSeconds = ($endTine - $startTime).TotalSeconds
87+
$results += $result
88+
}
89+
}
90+
91+
$reportPath = Join-Path $RepoRoot "artifacts" "AnalyseReport-$MatrixKey.json"
92+
$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8

‎.azure-pipelines/PipelineSteps/BatchGeneration/batch-generate-modules.ps1

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
param (
2+
[string]$MatrixKey,
3+
[string]$RepoRoot,
4+
[string]$AutorestVersion
5+
)
6+
7+
$generationTargetsOutputFile = Join-Path $RepoRoot "artifacts" "generationTargets.json"
8+
$generationTargets = Get-Content -Path $generationTargetsOutPutFile -Raw | ConvertFrom-Json
9+
$moduleGroup = $generationTargets.$MatrixKey
10+
Write-Host "##[group]Generating module group $MatrixKey"
11+
foreach ($key in $moduleGroup.PSObject.Properties.Name | Sort-Object) {
12+
$values = $moduleGroup.$key -join ', '
13+
Write-Output "$key : $values"
14+
}
15+
Write-Host "##[endgroup]"
16+
Write-Host
17+
$sortedModuleNames = $moduleGroup.PSObject.Properties.Name | Sort-Object
18+
19+
$AutorestOutputDir = Join-Path $RepoRoot "artifacts" "autorest"
20+
New-Item -ItemType Directory -Force -Path $AutorestOutputDir
21+
22+
$sourceDirectory = Join-Path $RepoRoot "src"
23+
$generatedDirectory = Join-Path $RepoRoot "generated"
24+
$buildScriptsModulePath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildScripts.psm1'
25+
Import-Module $buildScriptsModulePath -Force
26+
27+
$results = @()
28+
29+
foreach ($moduleName in $sortedModuleNames) {
30+
Write-Host "=============================================================="
31+
Write-Host "Regenerating Module: $moduleName"
32+
$moduleStartTime = Get-Date
33+
$moduleResult = @{
34+
Module = $moduleName
35+
DurationSeconds = 0
36+
Status = "Success"
37+
Changed = "No"
38+
SubModules = @()
39+
}
40+
41+
$subModuleNames = $moduleGroup.$moduleName
42+
foreach ($subModuleName in $subModuleNames) {
43+
Write-Host "Regenerating SubModule: $subModuleName"
44+
$subModuleStartTime = Get-Date
45+
$subModuleResult = @{
46+
MatrixKey = $MatrixKey
47+
SubModule = $subModuleName
48+
Status = "Success"
49+
DurationSeconds = 0
50+
Error = ""
51+
}
52+
53+
try {
54+
$generateLog = Join-Path $AutorestOutputDir $moduleName "$subModuleName.log"
55+
if (Test-Path $generateLog) {
56+
Remove-Item -Path $generateLog -Recurse -Force
57+
}
58+
New-Item -ItemType File -Force -Path $generateLog
59+
if (-not (Update-GeneratedSubModule -ModuleRootName $moduleName -SubModuleName $subModuleName -SourceDirectory $sourceDirectory -GeneratedDirectory $generatedDirectory -GenerateLog $generateLog -IsInvokedByPipeline $true)) {
60+
Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
61+
Write-Warning "log can be found at $generateLog"
62+
$moduleResult.Status = "Failed"
63+
$subModuleResult.Status = "Failed"
64+
$subModuleResult.Error = "Update-GeneratedSubModule function returned false."
65+
}
66+
67+
} catch {
68+
Write-Warning "Failed to regenerate module: $moduleName, sub module: $subModuleName"
69+
Write-Warning "Error message: $($_.Exception.Message)"
70+
$moduleResult.Status = "Failed"
71+
$subModuleResult.Status = "Failed"
72+
$subModuleResult.Error = $_.Exception.Message
73+
} finally {
74+
$subModuleEndTime = Get-Date
75+
$subModuleResult.DurationSeconds = ($subModuleEndTime - $subModuleStartTime).TotalSeconds
76+
$moduleResult.SubModules += $subModuleResult
77+
}
78+
}
79+
80+
# If the module is changed in either src or generated folder, add a change log entry
81+
Set-Location $RepoRoot
82+
$srcFolderModuleRelativePath = ".\src\$moduleName"
83+
$generatedFolderModuleRelativePath = ".\generated\$moduleName"
84+
$diffSrc = git diff --name-only HEAD -- $srcFolderModuleRelativePath
85+
$diffGenerated = git diff --name-only HEAD -- $generatedFolderModuleRelativePath
86+
$diff = $diffSrc -or $diffGenerated
87+
if ($diff) {
88+
Write-Host "Changes detected in $moduleName, adding change log"
89+
$moduleResult.Changed = "Yes"
90+
91+
$date = Get-Date -Format "yy-MM-dd"
92+
$newChangeLogEntry = "* Autorest version: $AutorestVersion - $date"
93+
94+
$updatedContent = @()
95+
$changeLogPath = Join-Path $RepoRoot "src" $moduleName $moduleName "AutorestUpgradeLog.md"
96+
97+
if (-not (Test-Path $changeLogPath)) {
98+
New-Item -Path $changeLogPath -ItemType File -Force | Out-Null
99+
$updatedContent += "## Autorest upgrade log"
100+
$updatedContent += $newChangeLogEntry
101+
} else{
102+
$changeLogContent = Get-Content -Path $changeLogPath
103+
$updatedContent += $changeLogContent[0]
104+
$updatedContent += $newChangeLogEntry
105+
$updatedContent += $changeLogContent[1..($changeLogContent.Count - 1)]
106+
}
107+
Set-Content $changeLogPath -Value $updatedContent
108+
109+
$moduleResult.Changed = "Yes, Autorest Change Log Updated"
110+
Write-Host "New change log entry added to $changeLogPath"
111+
}
112+
113+
$moduleEndTime = Get-Date
114+
$moduleResult.DurationSeconds = ($moduleEndTime - $moduleStartTime).TotalSeconds
115+
$results += $moduleResult
116+
}
117+
118+
$ArtifactOutputDir = Join-Path $RepoRoot "artifacts"
119+
Set-Location $RepoRoot
120+
121+
git add .
122+
$patchPath = Join-Path $ArtifactOutputDir "changed-$MatrixKey.patch"
123+
git diff --cached > $patchPath
124+
125+
$reportPath = Join-Path $ArtifactOutputDir "GenerationReport-$MatrixKey.json"
126+
$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8
127+
128+
Write-Host "Build report written to $reportPath"

‎.azure-pipelines/PipelineSteps/BatchGeneration/build-modules.ps1

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
param (
2+
[string]$MatrixKey,
3+
[string]$RepoRoot
4+
)
5+
6+
$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
7+
Import-Module $utilFilePath -Force
8+
$moduleGroup = Get-Targets -RepoRoot $RepoRoot -TargetsOutputFileName "buildTargets.json" -MatrixKey $MatrixKey
9+
$buildModulesPath = Join-Path $RepoRoot 'tools' 'BuildScripts' 'BuildModules.ps1'
10+
11+
$results = @()
12+
foreach ($moduleName in $moduleGroup) {
13+
Write-Host "=============================================================="
14+
Write-Host "Building Module: $moduleName"
15+
16+
$startTime = Get-Date
17+
$result = @{
18+
MatrixKey = $MatrixKey
19+
Module = $moduleName
20+
Status = "Success"
21+
DurationSeconds = 0
22+
Error = ""
23+
}
24+
25+
try {
26+
& $buildModulesPath -TargetModule $moduleName -InvokedByPipeline
27+
} catch {
28+
Write-Warning "Failed to build module: $moduleName"
29+
Write-Warning "Error message: $($_.Exception.Message)"
30+
$result.Status = "Failed"
31+
$result.Error = $_.Exception.Message
32+
} finally {
33+
$endTine = Get-Date
34+
$result.DurationSeconds = ($endTine - $startTime).TotalSeconds
35+
$results += $result
36+
}
37+
}
38+
39+
$reportPath = Join-Path $RepoRoot "artifacts" "BuildReport-$MatrixKey.json"
40+
$results | ConvertTo-Json -Depth 5 | Out-File -FilePath $reportPath -Encoding utf8

‎.azure-pipelines/PipelineSteps/BatchGeneration/create-branch.ps1

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
[CmdletBinding(DefaultParameterSetName="AllSet")]
2+
param (
3+
[string]$Owner,
4+
[string]$Repo,
5+
[string]$BaseBranch,
6+
[string]$NewBranch,
7+
[string]$Token
8+
)
9+
10+
$headers = @{ Authorization = "Bearer $Token"; "User-Agent" = "ADO-Pipeline" }
11+
$branchInfo = Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/ref/heads/$BaseBranch" -Headers $headers
12+
$sha = $branchInfo.object.sha
13+
14+
$body = @{
15+
ref = "refs/heads/$NewBranch"
16+
sha = $sha
17+
} | ConvertTo-Json
18+
19+
Invoke-RestMethod -Uri "https://api.github.com/repos/$Owner/$Repo/git/refs" `
20+
-Method Post -Headers $headers -Body $body -ContentType "application/json"
21+
22+
Write-Host "Created branch '$NewBranch' from '$BaseBranch'"

‎.azure-pipelines/PipelineSteps/BatchGeneration/filter.ps1

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
[CmdletBinding(DefaultParameterSetName="AllSet")]
2+
param (
3+
[int]$MaxParallelBuildJobs = 3,
4+
[int]$MaxParallelAnalyzeJobs = 3,
5+
[int]$MaxParallelTestWindowsJobs = 3,
6+
[int]$MaxParallelTestLinuxJobs = 3,
7+
[int]$MaxParallelTestMacJobs = 3,
8+
[string[]]$ChangedFiles,
9+
[string]$RepoRoot
10+
)
11+
12+
$utilFilePath = Join-Path $RepoRoot '.azure-pipelines' 'PipelineSteps' 'BatchGeneration' 'util.psm1'
13+
Import-Module $utilFilePath -Force
14+
$artifactsDir = Join-Path $RepoRoot 'artifacts'
15+
16+
$changedModulesDict = @{}
17+
$changedSubModulesDict = @{}
18+
if ($env:RUN_TEST_ON_ALL_MODULES -eq "True") {
19+
Write-Host "Run test on all modules"
20+
$V4ModulesFile = Join-Path $artifactsDir "generationTargets.json"
21+
$V4ModuleMaps = Get-Content -Raw -Path $V4ModulesFile | ConvertFrom-Json
22+
23+
foreach ($matrixKey in $V4ModuleMaps.PSObject.Properties.Name) {
24+
$moduleMap = $V4ModuleMaps.$matrixKey
25+
foreach ($moduleName in $moduleMap.PSObject.Properties.Name) {
26+
foreach ($subModuleName in $moduleMap.$moduleName) {
27+
$subModule = "$moduleName/$subModuleName"
28+
$changedModulesDict[$moduleName] = $true
29+
$changedSubModulesDict[$subModule] = $true
30+
}
31+
}
32+
}
33+
}
34+
else {
35+
Write-Host "Run test on generated folder changed modules"
36+
# Only generated filder change should trigger the test
37+
for ($i = 0; $i -lt $ChangedFiles.Count; $i++) {
38+
if ($ChangedFiles[$i] -match '^generated/([^/]+)/([^/]+\.autorest)/') {
39+
$moduleName = $Matches[2]
40+
$subModuleName = $Matches[3]
41+
$subModule = "$moduleName/$subModuleName"
42+
43+
$changedModulesDict[$moduleName] = $true
44+
$changedSubModulesDict[$subModule] = $true
45+
}
46+
}
47+
}
48+
49+
$changedModules = $changedModulesDict.Keys | Sort-Object
50+
$changedSubModules = $changedSubModulesDict.Keys | Sort-Object
51+
52+
Write-Host "##[group]Changed modules: $($changedModules.Count)"
53+
foreach ($module in $changedModules) {
54+
Write-Host $module
55+
}
56+
Write-Host "##[endgroup]"
57+
Write-Host
58+
59+
Write-Host "##[group]Changed sub modules: $($changedSubModules.Count)"
60+
foreach ($subModule in $changedSubModules) {
61+
Write-Host $subModule
62+
}
63+
Write-Host "##[endgroup]"
64+
Write-Host
65+
66+
$groupedBuildModules = Group-Modules -Modules $changedModules -MaxParallelJobs $MaxParallelBuildJobs
67+
Write-Matrix -GroupedModules $groupedBuildModules -VariableName 'buildTargets' -RepoRoot $RepoRoot
68+
69+
$groupedAnalyzeModules = Group-Modules -Modules $changedModules -MaxParallelJobs $MaxParallelAnalyzeJobs
70+
Write-Matrix -GroupedModules $groupedAnalyzeModules -VariableName 'analyzeTargets' -RepoRoot $RepoRoot
71+
72+
$groupedTestWindowsModules = Group-Modules -Modules $changedSubModules -MaxParallelJobs $MaxParallelTestWindowsJobs
73+
Write-Matrix -GroupedModules $groupedTestWindowsModules -VariableName 'testWindowsTargets' -RepoRoot $RepoRoot
74+
75+
$groupedTestLinuxModules = Group-Modules -Modules $changedSubModules -MaxParallelJobs $MaxParallelTestLinuxJobs
76+
Write-Matrix -GroupedModules $groupedTestLinuxModules -VariableName 'testLinuxTargets' -RepoRoot $RepoRoot
77+
78+
$groupedTestMacModules = Group-Modules -Modules $changedSubModules -MaxParallelJobs $MaxParallelTestMacJobs
79+
Write-Matrix -GroupedModules $groupedTestMacModules -VariableName 'testMacOSTargets' -RepoRoot $RepoRoot

0 commit comments

Comments
 (0)