Skip to content

Commit 498ee9b

Browse files
hvitvedhvitved
committed
C#: Factor C++ parts out of autobuilder
1 parent bb9c888 commit 498ee9b

32 files changed

+721
-288
lines changed

‎cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/BuildScripts.cs

Lines changed: 296 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,296 @@
1+
using Xunit;
2+
using Semmle.Autobuild.Shared;
3+
using System.Collections.Generic;
4+
using System;
5+
using System.Linq;
6+
using Microsoft.Build.Construction;
7+
using System.Xml;
8+
9+
namespace Semmle.Autobuild.Cpp.Tests
10+
{
11+
/// <summary>
12+
/// Test class to script Autobuilder scenarios.
13+
/// For most methods, it uses two fields:
14+
/// - an IList to capture the the arguments passed to it
15+
/// - an IDictionary of possible return values.
16+
/// </summary>
17+
class TestActions : IBuildActions
18+
{
19+
/// <summary>
20+
/// List of strings passed to FileDelete.
21+
/// </summary>
22+
public IList<string> FileDeleteIn = new List<string>();
23+
24+
void IBuildActions.FileDelete(string file)
25+
{
26+
FileDeleteIn.Add(file);
27+
}
28+
29+
public IList<string> FileExistsIn = new List<string>();
30+
public IDictionary<string, bool> FileExists = new Dictionary<string, bool>();
31+
32+
bool IBuildActions.FileExists(string file)
33+
{
34+
FileExistsIn.Add(file);
35+
if (FileExists.TryGetValue(file, out var ret))
36+
return ret;
37+
if (FileExists.TryGetValue(System.IO.Path.GetFileName(file), out ret))
38+
return ret;
39+
throw new ArgumentException("Missing FileExists " + file);
40+
}
41+
42+
public IList<string> RunProcessIn = new List<string>();
43+
public IDictionary<string, int> RunProcess = new Dictionary<string, int>();
44+
public IDictionary<string, string> RunProcessOut = new Dictionary<string, string>();
45+
public IDictionary<string, string> RunProcessWorkingDirectory = new Dictionary<string, string>();
46+
47+
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env, out IList<string> stdOut)
48+
{
49+
var pattern = cmd + " " + args;
50+
RunProcessIn.Add(pattern);
51+
if (RunProcessOut.TryGetValue(pattern, out var str))
52+
stdOut = str.Split("\n");
53+
else
54+
throw new ArgumentException("Missing RunProcessOut " + pattern);
55+
RunProcessWorkingDirectory.TryGetValue(pattern, out var wd);
56+
if (wd != workingDirectory)
57+
throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern);
58+
if (RunProcess.TryGetValue(pattern, out var ret))
59+
return ret;
60+
throw new ArgumentException("Missing RunProcess " + pattern);
61+
}
62+
63+
int IBuildActions.RunProcess(string cmd, string args, string? workingDirectory, IDictionary<string, string>? env)
64+
{
65+
var pattern = cmd + " " + args;
66+
RunProcessIn.Add(pattern);
67+
RunProcessWorkingDirectory.TryGetValue(pattern, out var wd);
68+
if (wd != workingDirectory)
69+
throw new ArgumentException("Missing RunProcessWorkingDirectory " + pattern);
70+
if (RunProcess.TryGetValue(pattern, out var ret))
71+
return ret;
72+
throw new ArgumentException("Missing RunProcess " + pattern);
73+
}
74+
75+
public IList<string> DirectoryDeleteIn = new List<string>();
76+
77+
void IBuildActions.DirectoryDelete(string dir, bool recursive)
78+
{
79+
DirectoryDeleteIn.Add(dir);
80+
}
81+
82+
public IDictionary<string, bool> DirectoryExists = new Dictionary<string, bool>();
83+
public IList<string> DirectoryExistsIn = new List<string>();
84+
85+
bool IBuildActions.DirectoryExists(string dir)
86+
{
87+
DirectoryExistsIn.Add(dir);
88+
if (DirectoryExists.TryGetValue(dir, out var ret))
89+
return ret;
90+
throw new ArgumentException("Missing DirectoryExists " + dir);
91+
}
92+
93+
public IDictionary<string, string?> GetEnvironmentVariable = new Dictionary<string, string?>();
94+
95+
string? IBuildActions.GetEnvironmentVariable(string name)
96+
{
97+
if (GetEnvironmentVariable.TryGetValue(name, out var ret))
98+
return ret;
99+
throw new ArgumentException("Missing GetEnvironmentVariable " + name);
100+
}
101+
102+
public string GetCurrentDirectory = "";
103+
104+
string IBuildActions.GetCurrentDirectory()
105+
{
106+
return GetCurrentDirectory;
107+
}
108+
109+
public IDictionary<string, string> EnumerateFiles = new Dictionary<string, string>();
110+
111+
IEnumerable<string> IBuildActions.EnumerateFiles(string dir)
112+
{
113+
if (EnumerateFiles.TryGetValue(dir, out var str))
114+
return str.Split("\n");
115+
throw new ArgumentException("Missing EnumerateFiles " + dir);
116+
}
117+
118+
public IDictionary<string, string> EnumerateDirectories = new Dictionary<string, string>();
119+
120+
IEnumerable<string> IBuildActions.EnumerateDirectories(string dir)
121+
{
122+
if (EnumerateDirectories.TryGetValue(dir, out var str))
123+
return string.IsNullOrEmpty(str) ? Enumerable.Empty<string>() : str.Split("\n");
124+
throw new ArgumentException("Missing EnumerateDirectories " + dir);
125+
}
126+
127+
public bool IsWindows;
128+
129+
bool IBuildActions.IsWindows() => IsWindows;
130+
131+
string IBuildActions.PathCombine(params string[] parts)
132+
{
133+
return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p)));
134+
}
135+
136+
string IBuildActions.GetFullPath(string path) => path;
137+
138+
void IBuildActions.WriteAllText(string filename, string contents)
139+
{
140+
}
141+
142+
public IDictionary<string, XmlDocument> LoadXml = new Dictionary<string, XmlDocument>();
143+
XmlDocument IBuildActions.LoadXml(string filename)
144+
{
145+
if (LoadXml.TryGetValue(filename, out var xml))
146+
return xml;
147+
throw new ArgumentException("Missing LoadXml " + filename);
148+
}
149+
150+
public string EnvironmentExpandEnvironmentVariables(string s)
151+
{
152+
foreach (var kvp in GetEnvironmentVariable)
153+
s = s.Replace($"%{kvp.Key}%", kvp.Value);
154+
return s;
155+
}
156+
}
157+
158+
/// <summary>
159+
/// A fake solution to build.
160+
/// </summary>
161+
class TestSolution : ISolution
162+
{
163+
public IEnumerable<SolutionConfigurationInSolution> Configurations => throw new NotImplementedException();
164+
165+
public string DefaultConfigurationName => "Release";
166+
167+
public string DefaultPlatformName => "x86";
168+
169+
public string FullPath { get; set; }
170+
171+
public Version ToolsVersion => new Version("14.0");
172+
173+
public IEnumerable<IProjectOrSolution> IncludedProjects => throw new NotImplementedException();
174+
175+
public TestSolution(string path)
176+
{
177+
FullPath = path;
178+
}
179+
}
180+
181+
public class BuildScriptTests
182+
{
183+
TestActions Actions = new TestActions();
184+
185+
// Records the arguments passed to StartCallback.
186+
IList<string> StartCallbackIn = new List<string>();
187+
188+
void StartCallback(string s, bool silent)
189+
{
190+
StartCallbackIn.Add(s);
191+
}
192+
193+
// Records the arguments passed to EndCallback
194+
IList<string> EndCallbackIn = new List<string>();
195+
IList<int> EndCallbackReturn = new List<int>();
196+
197+
void EndCallback(int ret, string s, bool silent)
198+
{
199+
EndCallbackReturn.Add(ret);
200+
EndCallbackIn.Add(s);
201+
}
202+
203+
CppAutobuilder CreateAutoBuilder(bool isWindows,
204+
string? buildless = null, string? solution = null, string? buildCommand = null, string? ignoreErrors = null,
205+
string? msBuildArguments = null, string? msBuildPlatform = null, string? msBuildConfiguration = null, string? msBuildTarget = null,
206+
string? dotnetArguments = null, string? dotnetVersion = null, string? vsToolsVersion = null,
207+
string? nugetRestore = null, string? allSolutions = null,
208+
string cwd = @"C:\Project")
209+
{
210+
string codeqlUpperLanguage = Language.Cpp.UpperCaseName;
211+
Actions.GetEnvironmentVariable[$"CODEQL_AUTOBUILDER_{codeqlUpperLanguage}_NO_INDEXING"] = "false";
212+
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_TRAP_DIR"] = "";
213+
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_SOURCE_ARCHIVE_DIR"] = "";
214+
Actions.GetEnvironmentVariable[$"CODEQL_EXTRACTOR_{codeqlUpperLanguage}_ROOT"] = $@"C:\codeql\{codeqlUpperLanguage.ToLowerInvariant()}";
215+
Actions.GetEnvironmentVariable["CODEQL_JAVA_HOME"] = @"C:\codeql\tools\java";
216+
Actions.GetEnvironmentVariable["SEMMLE_DIST"] = @"C:\odasa";
217+
Actions.GetEnvironmentVariable["SEMMLE_JAVA_HOME"] = @"C:\odasa\tools\java";
218+
Actions.GetEnvironmentVariable["SEMMLE_PLATFORM_TOOLS"] = @"C:\odasa\tools";
219+
Actions.GetEnvironmentVariable["LGTM_INDEX_VSTOOLS_VERSION"] = vsToolsVersion;
220+
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_ARGUMENTS"] = msBuildArguments;
221+
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_PLATFORM"] = msBuildPlatform;
222+
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_CONFIGURATION"] = msBuildConfiguration;
223+
Actions.GetEnvironmentVariable["LGTM_INDEX_MSBUILD_TARGET"] = msBuildTarget;
224+
Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_ARGUMENTS"] = dotnetArguments;
225+
Actions.GetEnvironmentVariable["LGTM_INDEX_DOTNET_VERSION"] = dotnetVersion;
226+
Actions.GetEnvironmentVariable["LGTM_INDEX_BUILD_COMMAND"] = buildCommand;
227+
Actions.GetEnvironmentVariable["LGTM_INDEX_SOLUTION"] = solution;
228+
Actions.GetEnvironmentVariable["LGTM_INDEX_IGNORE_ERRORS"] = ignoreErrors;
229+
Actions.GetEnvironmentVariable["LGTM_INDEX_BUILDLESS"] = buildless;
230+
Actions.GetEnvironmentVariable["LGTM_INDEX_ALL_SOLUTIONS"] = allSolutions;
231+
Actions.GetEnvironmentVariable["LGTM_INDEX_NUGET_RESTORE"] = nugetRestore;
232+
Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = isWindows ? @"C:\Program Files (x86)" : null;
233+
Actions.GetCurrentDirectory = cwd;
234+
Actions.IsWindows = isWindows;
235+
236+
var options = new AutobuildOptions(Actions, Language.Cpp);
237+
return new CppAutobuilder(Actions, options);
238+
}
239+
240+
void TestAutobuilderScript(Autobuilder autobuilder, int expectedOutput, int commandsRun)
241+
{
242+
Assert.Equal(expectedOutput, autobuilder.GetBuildScript().Run(Actions, StartCallback, EndCallback));
243+
244+
// Check expected commands actually ran
245+
Assert.Equal(commandsRun, StartCallbackIn.Count);
246+
Assert.Equal(commandsRun, EndCallbackIn.Count);
247+
Assert.Equal(commandsRun, EndCallbackReturn.Count);
248+
249+
var action = Actions.RunProcess.GetEnumerator();
250+
for (int cmd = 0; cmd < commandsRun; ++cmd)
251+
{
252+
Assert.True(action.MoveNext());
253+
254+
Assert.Equal(action.Current.Key, StartCallbackIn[cmd]);
255+
Assert.Equal(action.Current.Value, EndCallbackReturn[cmd]);
256+
}
257+
}
258+
259+
260+
[Fact]
261+
public void TestDefaultCppAutobuilder()
262+
{
263+
Actions.EnumerateFiles[@"C:\Project"] = "";
264+
Actions.EnumerateDirectories[@"C:\Project"] = "";
265+
266+
var autobuilder = CreateAutoBuilder(true);
267+
var script = autobuilder.GetBuildScript();
268+
269+
// Fails due to no solutions present.
270+
Assert.NotEqual(0, script.Run(Actions, StartCallback, EndCallback));
271+
}
272+
273+
[Fact]
274+
public void TestCppAutobuilderSuccess()
275+
{
276+
Actions.RunProcess[@"cmd.exe /C C:\odasa\tools\csharp\nuget\nuget.exe restore C:\Project\test.sln"] = 1;
277+
Actions.RunProcess[@"cmd.exe /C CALL ^""C:\Program Files ^(x86^)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat^"" && set Platform=&& type NUL && C:\odasa\tools\odasa index --auto msbuild C:\Project\test.sln /p:UseSharedCompilation=false /t:rebuild /p:Platform=""x86"" /p:Configuration=""Release"" /p:MvcBuildViews=true"] = 0;
278+
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = "";
279+
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 1;
280+
Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = 0;
281+
Actions.RunProcessOut[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationVersion"] = "";
282+
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = true;
283+
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall.bat"] = true;
284+
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 11.0\VC\vcvarsall.bat"] = true;
285+
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\vcvarsall.bat"] = true;
286+
Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true;
287+
Actions.EnumerateFiles[@"C:\Project"] = "foo.cs\ntest.slx";
288+
Actions.EnumerateDirectories[@"C:\Project"] = "";
289+
290+
var autobuilder = CreateAutoBuilder(true);
291+
var solution = new TestSolution(@"C:\Project\test.sln");
292+
autobuilder.ProjectsOrSolutionsToBuild.Add(solution);
293+
TestAutobuilderScript(autobuilder, 0, 2);
294+
}
295+
}
296+
}

‎cpp/autobuilder/Semmle.Autobuild.Cpp.Tests/Semmle.Autobuild.Cpp.Tests.csproj

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>netcoreapp3.0</TargetFramework>
6+
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
7+
<RuntimeIdentifiers>win-x64;linux-x64;osx-x64</RuntimeIdentifiers>
8+
<Nullable>enable</Nullable>
9+
</PropertyGroup>
10+
11+
<ItemGroup>
12+
<PackageReference Include="System.IO.FileSystem" Version="4.3.0" />
13+
<PackageReference Include="System.IO.FileSystem.Primitives" Version="4.3.0" />
14+
<PackageReference Include="xunit" Version="2.4.1" />
15+
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1">
16+
<PrivateAssets>all</PrivateAssets>
17+
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
18+
</PackageReference>
19+
</ItemGroup>
20+
21+
<ItemGroup>
22+
<ProjectReference Include="..\Semmle.Autobuild.Cpp\Semmle.Autobuild.Cpp.csproj" />
23+
<ProjectReference Include="..\..\..\csharp\autobuilder\Semmle.Autobuild.Shared\Semmle.Autobuild.Shared.csproj" />
24+
</ItemGroup>
25+
</Project>

‎cpp/autobuilder/Semmle.Autobuild.Cpp/CppAutobuilder.cs

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
using Semmle.Autobuild.Shared;
2+
3+
namespace Semmle.Autobuild.Cpp
4+
{
5+
public class CppAutobuilder : Autobuilder
6+
{
7+
public CppAutobuilder(IBuildActions actions, AutobuildOptions options) : base(actions, options) { }
8+
9+
public override BuildScript GetBuildScript()
10+
{
11+
if (Options.BuildCommand != null)
12+
return new BuildCommandRule((_, f) => f(null)).Analyse(this, false);
13+
14+
return
15+
// First try MSBuild
16+
new MsBuildRule().Analyse(this, true) |
17+
// Then look for a script that might be a build script
18+
(() => new BuildCommandAutoRule((_, f) => f(null)).Analyse(this, true)) |
19+
// All attempts failed: print message
20+
AutobuildFailure();
21+
}
22+
}
23+
}

‎csharp/autobuilder/Semmle.Autobuild/Program.cs renamed to ‎cpp/autobuilder/Semmle.Autobuild.Cpp/Program.cs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
2+
using Semmle.Autobuild.Shared;
23

3-
namespace Semmle.Autobuild
4+
namespace Semmle.Autobuild.Cpp
45
{
56
class Program
67
{
@@ -10,11 +11,11 @@ static int Main()
1011
try
1112
{
1213
var actions = SystemBuildActions.Instance;
13-
var options = new AutobuildOptions(actions);
14+
var options = new AutobuildOptions(actions, Language.Cpp);
1415
try
1516
{
16-
Console.WriteLine($"Semmle autobuilder for {options.Language}");
17-
var builder = new Autobuilder(actions, options);
17+
Console.WriteLine("CodeQL C++ autobuilder");
18+
var builder = new CppAutobuilder(actions, options);
1819
return builder.AttemptBuild();
1920
}
2021
catch(InvalidEnvironmentException ex)

‎csharp/autobuilder/Semmle.Autobuild.Tests/Properties/AssemblyInfo.cs renamed to ‎cpp/autobuilder/Semmle.Autobuild.Cpp/Properties/AssemblyInfo.cs

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,12 @@
44
// General Information about an assembly is controlled through the following
55
// set of attributes. Change these attribute values to modify the information
66
// associated with an assembly.
7-
[assembly: AssemblyTitle("Semmle.Autobuild.Tests")]
7+
[assembly: AssemblyTitle("Semmle.Autobuild.Cpp")]
88
[assembly: AssemblyDescription("")]
99
[assembly: AssemblyConfiguration("")]
10-
[assembly: AssemblyCompany("")]
11-
[assembly: AssemblyProduct("Semmle.Extraction.Tests")]
12-
[assembly: AssemblyCopyright("Copyright © 2018")]
10+
[assembly: AssemblyCompany("GitHub")]
11+
[assembly: AssemblyProduct("CodeQL autobuilder for C++")]
12+
[assembly: AssemblyCopyright("Copyright © GitHub 2020")]
1313
[assembly: AssemblyTrademark("")]
1414
[assembly: AssemblyCulture("")]
1515

0 commit comments

Comments
 (0)