Skip to content

Commit 29030b4

Browse files
mthalmanmthalman
authored
CLI tool to update dependencies (microsoft#460)
1 parent 5255f27 commit 29030b4

15 files changed

+765
-0
lines changed

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,6 @@
2121

2222
# ImageBuilder directory
2323
.Microsoft.DotNet.ImageBuilder
24+
25+
# Visual Studio debug profile
26+
**/launchSettings.json

‎eng/lcu-info.json

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
[
2+
{
3+
"osVersion": "windowsservercore-ltsc2016",
4+
"runtimeVersions": [
5+
"3.5",
6+
"4.7",
7+
"4.7.1",
8+
"4.7.2"
9+
],
10+
"downloadUrl": "http://download.windowsupdate.com/d/msdownload/update/software/secu/2019/11/windows10.0-kb4525236-x64_6d152dcde0058b5771eba1055df7dd719fc5faef.msu"
11+
},
12+
{
13+
"osVersion": "windowsservercore-ltsc2016",
14+
"runtimeVersions": [
15+
"4.8"
16+
],
17+
"downloadUrl": "http://download.windowsupdate.com/c/msdownload/update/software/updt/2019/09/windows10.0-kb4515839-x64_52b3515c58cac8149b662f0486b553f176f607a5.msu"
18+
},
19+
{
20+
"osVersion": "windowsservercore-ltsc2019",
21+
"runtimeVersions": [
22+
"3.5"
23+
],
24+
"downloadUrl": "http://download.windowsupdate.com/c/msdownload/update/software/updt/2019/09/windows10.0-kb4515843-x64_181da0224818b03254ff48178c3cd7f73501c9db.msu"
25+
},
26+
{
27+
"osVersion": "windowsservercore-ltsc2019",
28+
"runtimeVersions": [
29+
"4.8"
30+
],
31+
"downloadUrl": "http://download.windowsupdate.com/c/msdownload/update/software/updt/2019/09/windows10.0-kb4515843-x64_181da0224818b03254ff48178c3cd7f73501c9db.msu"
32+
},
33+
{
34+
"osVersion": "windowsservercore-1903",
35+
"runtimeVersions": [
36+
"3.5"
37+
],
38+
"downloadUrl": "http://download.windowsupdate.com/d/msdownload/update/software/updt/2019/09/windows10.0-kb4515871-x64_03a6aec54b0ca2fc40efcff09a810064f5d2ae60.msu"
39+
}
40+
]

‎eng/nuget-info.json

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
[
2+
{
3+
"nugetClientVersion": "4.4.3",
4+
"osVersions": [
5+
"windowsservercore-1903",
6+
"windowsservercore-ltsc2016",
7+
"windowsservercore-ltsc2019"
8+
]
9+
},
10+
{
11+
"nugetClientVersion": "5.3.1",
12+
"osVersions": [
13+
"windowsservercore-1909"
14+
]
15+
}
16+
]

‎eng/update-dependencies/CustomFileRegexUpdater.cs

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System.Collections.Generic;
6+
using System.Linq;
7+
using Microsoft.DotNet.VersionTools.Dependencies;
8+
9+
namespace Microsoft.DotNet.Framework.UpdateDependencies
10+
{
11+
public class CustomFileRegexUpdater : FileRegexUpdater
12+
{
13+
private readonly string replacementValue;
14+
private readonly string buildInfoName;
15+
16+
public CustomFileRegexUpdater(string replacementValue, string buildInfoName)
17+
{
18+
this.replacementValue = replacementValue;
19+
this.buildInfoName = buildInfoName;
20+
}
21+
22+
protected override string TryGetDesiredValue(IEnumerable<IDependencyInfo> dependencyInfos, out IEnumerable<IDependencyInfo> usedDependencyInfos)
23+
{
24+
usedDependencyInfos = dependencyInfos.Where(info => info.SimpleName == this.buildInfoName);
25+
26+
return this.replacementValue;
27+
}
28+
}
29+
}

‎eng/update-dependencies/DependencyUpdater.cs

+302
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,302 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
using System.Collections.Generic;
7+
using System.Diagnostics;
8+
using System.IO;
9+
using System.Linq;
10+
using System.Text.RegularExpressions;
11+
using System.Threading.Tasks;
12+
using Microsoft.DotNet.Framework.UpdateDependencies.Models;
13+
using Microsoft.DotNet.VersionTools;
14+
using Microsoft.DotNet.VersionTools.Automation;
15+
using Microsoft.DotNet.VersionTools.Dependencies;
16+
using Microsoft.DotNet.VersionTools.Dependencies.BuildOutput;
17+
using Newtonsoft.Json;
18+
19+
namespace Microsoft.DotNet.Framework.UpdateDependencies
20+
{
21+
public class DependencyUpdater
22+
{
23+
private readonly Options options;
24+
private static readonly Lazy<IEnumerable<DockerfileInfo>> dockerfiles;
25+
26+
public const string RuntimeImageVariant = "runtime";
27+
public const string SdkImageVariant = "sdk";
28+
public const string AspnetImageVariant = "aspnet";
29+
public const string WcfImageVariant = "wcf";
30+
31+
public DependencyUpdater(Options options)
32+
{
33+
this.options = options;
34+
}
35+
36+
static DependencyUpdater()
37+
{
38+
dockerfiles = new Lazy<IEnumerable<DockerfileInfo>>(() =>
39+
new DirectoryInfo(Program.RepoRoot).GetDirectories("4.*")
40+
.Append(new DirectoryInfo(Path.Combine(Program.RepoRoot, "3.5")))
41+
.SelectMany(dir => dir.GetFiles("Dockerfile", SearchOption.AllDirectories))
42+
.Select(file => new DockerfileInfo(file.FullName))
43+
.ToArray());
44+
}
45+
46+
public async Task ExecuteAsync()
47+
{
48+
IEnumerable<IDependencyInfo> dependencyInfos = new IDependencyInfo[]
49+
{
50+
CreateBuildInfo(RuntimeImageVariant,
51+
this.options.DateStampRuntime?? this.options.DateStampAll ?? String.Empty),
52+
CreateBuildInfo(SdkImageVariant,
53+
this.options.DateStampSdk ?? this.options.DateStampAll ?? String.Empty),
54+
CreateBuildInfo(AspnetImageVariant,
55+
this.options.DateStampAspnet ?? this.options.DateStampAll ?? String.Empty),
56+
CreateBuildInfo(WcfImageVariant,
57+
this.options.DateStampWcf ?? this.options.DateStampAll ?? String.Empty),
58+
};
59+
60+
DependencyUpdateResults updateResults = UpdateFiles(dependencyInfos);
61+
if (updateResults.ChangesDetected())
62+
{
63+
if (this.options.UpdateOnly)
64+
{
65+
Trace.TraceInformation($"Changes made but no GitHub credentials specified, skipping PR creation");
66+
}
67+
else
68+
{
69+
await CreatePullRequestAsync(dependencyInfos);
70+
}
71+
}
72+
}
73+
74+
private async Task CreatePullRequestAsync(IEnumerable<IDependencyInfo> buildInfos)
75+
{
76+
GitHubAuth gitHubAuth = new GitHubAuth(this.options.GitHubPassword, this.options.GitHubUser, this.options.GitHubEmail);
77+
PullRequestCreator prCreator = new PullRequestCreator(gitHubAuth, this.options.GitHubUser);
78+
PullRequestOptions prOptions = new PullRequestOptions()
79+
{
80+
BranchNamingStrategy = new SingleBranchNamingStrategy($"UpdateDependencies-{this.options.GitHubUpstreamBranch}")
81+
};
82+
83+
string commitMessage = $"[{this.options.GitHubUpstreamBranch}] Update image dependencies";
84+
85+
await prCreator.CreateOrUpdateAsync(
86+
commitMessage,
87+
commitMessage,
88+
string.Empty,
89+
new GitHubBranch(this.options.GitHubUpstreamBranch, new GitHubProject(this.options.GitHubProject, this.options.GitHubUpstreamOwner)),
90+
new GitHubProject(this.options.GitHubProject, gitHubAuth.User),
91+
prOptions);
92+
}
93+
94+
private static BuildDependencyInfo CreateBuildInfo(string name, string version)
95+
{
96+
return new BuildDependencyInfo(
97+
new BuildInfo
98+
{
99+
Name = name,
100+
LatestPackages = new Dictionary<string, string> { },
101+
LatestReleaseVersion = version
102+
},
103+
false,
104+
Enumerable.Empty<string>());
105+
}
106+
107+
private DependencyUpdateResults UpdateFiles(IEnumerable<IDependencyInfo> buildInfos)
108+
{
109+
List<IDependencyUpdater> updaters = new List<IDependencyUpdater>();
110+
111+
updaters.AddRange(CreateManifestUpdaters());
112+
updaters.AddRange(CreateLcuUpdaters());
113+
updaters.AddRange(CreateVsUpdaters());
114+
updaters.AddRange(CreateNuGetUpdaters());
115+
updaters.Add(new ReadmeUpdater());
116+
117+
return DependencyUpdateUtils.Update(updaters, buildInfos);
118+
}
119+
120+
private IEnumerable<IDependencyUpdater> CreateLcuUpdaters()
121+
{
122+
LcuInfo[] lcuConfigs = JsonConvert.DeserializeObject<LcuInfo[]>(
123+
File.ReadAllText(this.options.LcuInfoPath));
124+
125+
const string UrlVersionGroupName = "Url";
126+
const string CabFileVersionGroupName = "CabFile";
127+
128+
return dockerfiles.Value
129+
.Where(dockerfile => dockerfile.ImageVariant == RuntimeImageVariant)
130+
.Select(dockerfile => new
131+
{
132+
Dockerfile = dockerfile,
133+
LcuConfigInfo = GetLcuConfigInfo(dockerfile, lcuConfigs)
134+
})
135+
.Where(val => val.LcuConfigInfo != null)
136+
.SelectMany(val =>
137+
new IDependencyUpdater[]
138+
{
139+
new CustomFileRegexUpdater(val.LcuConfigInfo!.DownloadUrl, RuntimeImageVariant)
140+
{
141+
Path = val.Dockerfile.Path,
142+
VersionGroupName = UrlVersionGroupName,
143+
Regex = new Regex(@$"# Apply latest patch(.|\n)+(?<{UrlVersionGroupName}>http:\/\/[^\s""]+)")
144+
},
145+
new CustomFileRegexUpdater(ParseCabFileName(val.LcuConfigInfo!.DownloadUrl), RuntimeImageVariant)
146+
{
147+
Path = val.Dockerfile.Path,
148+
VersionGroupName = CabFileVersionGroupName,
149+
Regex = new Regex(@$"# Apply latest patch(.|\n)+dism.+C:\\patch\\(?<{CabFileVersionGroupName}>\S+)", RegexOptions.IgnoreCase)
150+
}
151+
}
152+
);
153+
}
154+
155+
private IEnumerable<IDependencyUpdater> CreateVsUpdaters()
156+
{
157+
VsInfo vsConfig = JsonConvert.DeserializeObject<VsInfo>(
158+
File.ReadAllText(this.options.VsInfoPath));
159+
160+
const string TestAgentGroupName = "TestAgent";
161+
const string BuildToolsGroupName = "BuildTools";
162+
const string WebTargetsGroupName = "WebTargets";
163+
164+
return dockerfiles.Value
165+
.Where(dockerfile => dockerfile.ImageVariant == SdkImageVariant)
166+
.SelectMany(dockerfile =>
167+
new IDependencyUpdater[]
168+
{
169+
new CustomFileRegexUpdater(vsConfig.TestAgentUrl, dockerfile.ImageVariant)
170+
{
171+
VersionGroupName = TestAgentGroupName,
172+
Path = dockerfile.Path,
173+
Regex = new Regex(@$"(?<{TestAgentGroupName}>https:\/\/\S+vs_TestAgent\.exe)"),
174+
},
175+
new CustomFileRegexUpdater(vsConfig.BuildToolsUrl, dockerfile.ImageVariant)
176+
{
177+
VersionGroupName = BuildToolsGroupName,
178+
Path = dockerfile.Path,
179+
Regex = new Regex(@$"(?<{BuildToolsGroupName}>https:\/\/\S+vs_BuildTools\.exe)"),
180+
},
181+
new CustomFileRegexUpdater(vsConfig.WebTargetsUrl, dockerfile.ImageVariant)
182+
{
183+
VersionGroupName = WebTargetsGroupName,
184+
Path = dockerfile.Path,
185+
Regex = new Regex(@$"# Install web targets(.|\n)+?(?<{WebTargetsGroupName}>https:\/\/\S+)"),
186+
}
187+
});
188+
}
189+
190+
private static string ParseCabFileName(string lcuDownloadUrl)
191+
{
192+
string msuFilename = lcuDownloadUrl.Substring(lcuDownloadUrl.LastIndexOf("/") + 1);
193+
return msuFilename.Substring(0, msuFilename.IndexOf("_")) + ".cab";
194+
}
195+
196+
private static LcuInfo? GetLcuConfigInfo(DockerfileInfo dockerfile, LcuInfo[] lcuConfigs)
197+
{
198+
return lcuConfigs
199+
.FirstOrDefault(config => config.OsVersion == dockerfile.OsVersion &&
200+
config.RuntimeVersions.Any(runtime => runtime == dockerfile.FrameworkVersion));
201+
}
202+
203+
private IEnumerable<IDependencyUpdater> CreateNuGetUpdaters()
204+
{
205+
NuGetInfo[] nuGetVersions = JsonConvert.DeserializeObject<NuGetInfo[]>(
206+
File.ReadAllText(this.options.NuGetInfoPath));
207+
208+
const string NuGetVersionGroupName = "version";
209+
210+
return dockerfiles.Value
211+
.Where(dockerfile => dockerfile.ImageVariant == SdkImageVariant)
212+
.Select(dockerfile =>
213+
{
214+
// Find the NuGetInfo that matches the OS version of this Dockerfile
215+
NuGetInfo nuGetInfo = nuGetVersions.FirstOrDefault(ver => ver.OsVersions.Contains(dockerfile.OsVersion));
216+
if (nuGetInfo is null)
217+
{
218+
throw new InvalidOperationException($"No NuGet info is specified in '{this.options.NuGetInfoPath}' for OS version '{dockerfile.OsVersion}'.");
219+
}
220+
221+
return new CustomFileRegexUpdater(nuGetInfo.NuGetClientVersion, dockerfile.ImageVariant)
222+
{
223+
Path = dockerfile.Path,
224+
VersionGroupName = NuGetVersionGroupName,
225+
Regex = new Regex(@$"ENV NUGET_VERSION (?<{NuGetVersionGroupName}>\d+\.\d+\.\d+)")
226+
};
227+
});
228+
}
229+
230+
private IEnumerable<IDependencyUpdater> CreateManifestUpdaters()
231+
{
232+
const string RuntimePrefix = "Runtime";
233+
const string SdkPrefix = "Sdk";
234+
const string AspnetPrefix = "Aspnet";
235+
const string WcfPrefix = "Wcf";
236+
237+
if (this.options.DateStampAll != null)
238+
{
239+
yield return CreateManifestUpdater(RuntimePrefix, RuntimeImageVariant);
240+
yield return CreateManifestUpdater(SdkPrefix, SdkImageVariant);
241+
yield return CreateManifestUpdater(AspnetPrefix, AspnetImageVariant);
242+
yield return CreateManifestUpdater(WcfPrefix, WcfImageVariant);
243+
}
244+
else
245+
{
246+
if (this.options.DateStampRuntime != null)
247+
{
248+
yield return CreateManifestUpdater(RuntimePrefix, RuntimeImageVariant);
249+
}
250+
251+
if (this.options.DateStampSdk != null)
252+
{
253+
yield return CreateManifestUpdater(SdkPrefix, SdkImageVariant);
254+
}
255+
256+
if (this.options.DateStampAspnet != null)
257+
{
258+
yield return CreateManifestUpdater(AspnetPrefix, AspnetImageVariant);
259+
}
260+
261+
if (this.options.DateStampWcf != null)
262+
{
263+
yield return CreateManifestUpdater(WcfPrefix, WcfImageVariant);
264+
}
265+
}
266+
}
267+
268+
private static IDependencyUpdater CreateManifestUpdater(string dateStampVariablePrefix, string buildInfoName)
269+
{
270+
const string TagDateStampGroupName = "tagDateStampValue";
271+
272+
return new FileRegexReleaseUpdater
273+
{
274+
Path = Path.Combine(Program.RepoRoot, "manifest.json"),
275+
BuildInfoName = buildInfoName,
276+
Regex = new Regex($"\"{dateStampVariablePrefix}ReleaseDateStamp\": \"(?<{TagDateStampGroupName}>\\d{{8}})\""),
277+
VersionGroupName = TagDateStampGroupName
278+
};
279+
}
280+
281+
private class DockerfileInfo
282+
{
283+
public DockerfileInfo(string path)
284+
{
285+
this.Path = path;
286+
287+
string[] pathParts = path.Substring(Program.RepoRoot.Length + 1)
288+
.Replace(@"\", "/")
289+
.Split("/");
290+
291+
this.FrameworkVersion = pathParts[0];
292+
this.ImageVariant = pathParts[1];
293+
this.OsVersion = pathParts[2];
294+
}
295+
296+
public string Path { get; }
297+
public string FrameworkVersion { get; }
298+
public string OsVersion { get; }
299+
public string ImageVariant { get; }
300+
}
301+
}
302+
}

0 commit comments

Comments
 (0)