Skip to content

Commit 90ec876

Browse files
jkoritzinskytrylek
jkoritzinsky
and
trylek
authored
Convert JIT/Methodical/Array/range Pri0 tests and hook up mobile testing (#62752)
Co-authored-by: Tomas <[email protected]>
1 parent 0e800a6 commit 90ec876

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+683
-300
lines changed

‎eng/Version.Details.xml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,10 @@
238238
<Uri>https://github.com/dotnet/linker</Uri>
239239
<Sha>1a6468ff722c8d362f37638989ec01ab9975ac28</Sha>
240240
</Dependency>
241+
<Dependency Name="Microsoft.DotNet.XHarness.TestRunners.Common" Version="1.0.0-prerelease.22053.2">
242+
<Uri>https://github.com/dotnet/xharness</Uri>
243+
<Sha>7c293e5b676eb5b57418f775587badad65ec07f5</Sha>
244+
</Dependency>
241245
<Dependency Name="Microsoft.DotNet.XHarness.TestRunners.Xunit" Version="1.0.0-prerelease.22053.2">
242246
<Uri>https://github.com/dotnet/xharness</Uri>
243247
<Sha>7c293e5b676eb5b57418f775587badad65ec07f5</Sha>

‎eng/Versions.props

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@
160160
<!-- Testing -->
161161
<MicrosoftNETCoreCoreDisToolsVersion>1.0.1-prerelease-00006</MicrosoftNETCoreCoreDisToolsVersion>
162162
<MicrosoftNETTestSdkVersion>16.9.0-preview-20201201-01</MicrosoftNETTestSdkVersion>
163+
<MicrosoftDotNetXHarnessTestRunnersCommonVersion>1.0.0-prerelease.22053.2</MicrosoftDotNetXHarnessTestRunnersCommonVersion>
163164
<MicrosoftDotNetXHarnessTestRunnersXunitVersion>1.0.0-prerelease.22053.2</MicrosoftDotNetXHarnessTestRunnersXunitVersion>
164165
<MicrosoftDotNetXHarnessCLIVersion>1.0.0-prerelease.22053.2</MicrosoftDotNetXHarnessCLIVersion>
165166
<MicrosoftDotNetHotReloadUtilsGeneratorBuildToolVersion>1.0.2-alpha.0.22053.3</MicrosoftDotNetHotReloadUtilsGeneratorBuildToolVersion>

‎eng/testing/tests.ioslike.targets

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,6 @@
130130

131131
<Target Name="PrepareForAppleBuildApp">
132132
<Error Condition="!Exists('$(MicrosoftNetCoreAppRuntimePackRidDir)')" Text="MicrosoftNetCoreAppRuntimePackRidDir=$(MicrosoftNetCoreAppRuntimePackRidDir) doesn't exist" />
133-
<Error Condition="'$(TestArchiveTestsDir)' == ''" Text="TestArchiveTestsDir property to archive the test folder must be set." />
134133

135134
<WriteLinesToFile File="$(PublishDir)xunit-excludes.txt" Lines="$(XunitExcludesTxtFileContent)" Overwrite="true" />
136135

@@ -169,6 +168,7 @@
169168

170169
<Target Name="_CopyTestArchive"
171170
Condition="'$(ArchiveTests)' == 'true' and '$(IgnoreForCI)' != 'true'">
171+
<Error Condition="'$(TestArchiveTestsDir)' == ''" Text="TestArchiveTestsDir property to archive the test folder must be set." />
172172

173173
<!-- Adjust the variable names -->
174174
<PropertyGroup Condition="'$(GenerateXcodeProject)' == 'true'">

‎eng/testing/tests.mobile.targets

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,8 @@
198198
</Target>
199199

200200
<Target Name="AddTestRunnersToPublishedFiles"
201-
AfterTargets="ComputeResolvedFilesToPublishList">
201+
AfterTargets="ComputeResolvedFilesToPublishList"
202+
Condition="'$(TestFramework)' == 'xunit'">
202203
<ItemGroup>
203204
<_runnerFilesToPublish Include="$(AndroidTestRunnerDir)*" Condition="'$(TargetOS)' == 'Android'" />
204205
<_runnerFilesToPublish Include="$(AppleTestRunnerDir)*" Condition="'$(TargetOS)' == 'MacCatalyst' or '$(TargetOS)' == 'iOS' or '$(TargetOS)' == 'iOSSimulator' or '$(TargetOS)' == 'tvOS' or '$(TargetOS)' == 'tvOSSimulator'" />

‎eng/testing/tests.wasm.targets

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@
4747
<_XHarnessArgs Condition="'$(IsFunctionalTest)' == 'true'" >$(_XHarnessArgs) --expected-exit-code=$(ExpectedExitCode)</_XHarnessArgs>
4848
<_XHarnessArgs Condition="'$(WasmXHarnessArgs)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgs)</_XHarnessArgs>
4949

50-
<_AppArgs Condition="'$(IsFunctionalTest)' != 'true' and '$(Scenario)' != 'BuildWasmApps'">--run WasmTestRunner.dll $(AssemblyName).dll</_AppArgs>
50+
<_AppArgs Condition="'$(IsFunctionalTest)' != 'true' and '$(Scenario)' != 'BuildWasmApps' and '$(WasmMainAssemblyFileName)' == ''">--run WasmTestRunner.dll $(AssemblyName).dll</_AppArgs>
51+
<_AppArgs Condition="'$(IsFunctionalTest)' != 'true' and '$(WasmMainAssemblyFileName)' != ''">--run $(WasmMainAssemblyFileName)</_AppArgs>
5152
<_AppArgs Condition="'$(IsFunctionalTest)' == 'true'">--run $(AssemblyName).dll</_AppArgs>
5253

5354
<_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs)</_AppArgs>

‎src/tasks/WasmAppBuilder/WasmAppBuilder.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@ private bool ExecuteInternal ()
175175
return false;
176176
}
177177
var mainFileName=Path.GetFileName(MainJS);
178+
Log.LogMessage(MessageImportance.Low, $"MainJS path: '{MainJS}', fileName : '{mainFileName}', destination: '{Path.Combine(AppDir, mainFileName)}'");
178179
FileCopyChecked(MainJS!, Path.Combine(AppDir, mainFileName), string.Empty);
179180

180181
string indexHtmlPath = Path.Combine(AppDir, "index.html");

‎src/tests/Common/XHarnessRunnerLibrary/GeneratedTestRunner.cs

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Diagnostics;
4+
using System.IO;
5+
using System.Threading.Tasks;
6+
using Microsoft.DotNet.XHarness.Common;
7+
using Microsoft.DotNet.XHarness.TestRunners.Common;
8+
using XUnitWrapperLibrary;
9+
10+
namespace XHarnessRunnerLibrary;
11+
12+
public sealed class GeneratedTestRunner : TestRunner
13+
{
14+
string _assemblyName;
15+
TestFilter.ISearchClause? _filter;
16+
Func<TestFilter?, TestSummary> _runTestsCallback;
17+
public GeneratedTestRunner(LogWriter logger, Func<TestFilter?, TestSummary> runTestsCallback, string assemblyName)
18+
:base(logger)
19+
{
20+
_assemblyName = assemblyName;
21+
_runTestsCallback = runTestsCallback;
22+
ResultsFileName = $"{_assemblyName}.testResults.xml";
23+
}
24+
25+
public TestSummary LastTestRun { get; private set; } = new();
26+
27+
protected override string ResultsFileName { get; set; }
28+
29+
public override Task Run(IEnumerable<TestAssemblyInfo> testAssemblies)
30+
{
31+
LastTestRun = _runTestsCallback(_filter is not null ? new TestFilter(_filter) : null);
32+
PassedTests = LastTestRun.PassedTests;
33+
FailedTests = LastTestRun.FailedTests;
34+
SkippedTests = LastTestRun.SkippedTests;
35+
ExecutedTests = PassedTests + FailedTests;
36+
TotalTests = ExecutedTests + SkippedTests;
37+
return Task.CompletedTask;
38+
}
39+
40+
public override string WriteResultsToFile(XmlResultJargon xmlResultJargon)
41+
{
42+
Debug.Assert(xmlResultJargon == XmlResultJargon.xUnit);
43+
File.WriteAllText(ResultsFileName, LastTestRun.GetTestResultOutput(_assemblyName));
44+
return ResultsFileName;
45+
}
46+
47+
public override void WriteResultsToFile(TextWriter writer, XmlResultJargon jargon)
48+
{
49+
Debug.Assert(jargon == XmlResultJargon.xUnit);
50+
writer.WriteLine(LastTestRun.GetTestResultOutput(_assemblyName));
51+
}
52+
53+
public override void SkipTests(IEnumerable<string> tests)
54+
{
55+
foreach (string test in tests)
56+
{
57+
var testNameClause = new TestFilter.NotClause(new TestFilter.NameClause(TestFilter.TermKind.DisplayName, test, true));
58+
_filter = _filter is null ? testNameClause : new TestFilter.AndClause(_filter, testNameClause);
59+
}
60+
}
61+
62+
public override void SkipCategories(IEnumerable<string> categories)
63+
{
64+
}
65+
66+
public override void SkipMethod(string method, bool isExcluded)
67+
{
68+
TestFilter.ISearchClause methodClause = new TestFilter.NameClause(TestFilter.TermKind.FullyQualifiedName, method, true);
69+
if (isExcluded)
70+
{
71+
methodClause = new TestFilter.NotClause(methodClause);
72+
_filter = _filter is null ? methodClause : new TestFilter.AndClause(_filter, methodClause);
73+
}
74+
else
75+
{
76+
_filter = _filter is null ? methodClause : new TestFilter.OrClause(_filter, methodClause);
77+
}
78+
}
79+
80+
public override void SkipClass(string className, bool isExcluded)
81+
{
82+
throw new NotImplementedException();
83+
}
84+
}

‎src/tests/Common/XHarnessRunnerLibrary/RunnerEntryPoint.cs

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Threading.Tasks;
5+
using Microsoft.DotNet.XHarness.TestRunners.Common;
6+
using XUnitWrapperLibrary;
7+
8+
namespace XHarnessRunnerLibrary;
9+
10+
public static class RunnerEntryPoint
11+
{
12+
public static async Task<int> RunTests(Func<TestFilter?, TestSummary> runTestsCallback, string assemblyName, string? filter)
13+
{
14+
ApplicationEntryPoint? entryPoint = null;
15+
if (OperatingSystem.IsAndroid())
16+
{
17+
entryPoint = new AndroidEntryPoint(new SimpleDevice(assemblyName), runTestsCallback, assemblyName, filter);
18+
}
19+
if (OperatingSystem.IsMacCatalyst() || OperatingSystem.IsIOS() || OperatingSystem.IsTvOS() || OperatingSystem.IsWatchOS())
20+
{
21+
entryPoint = new AppleEntryPoint(new SimpleDevice(assemblyName), runTestsCallback, assemblyName, filter);
22+
}
23+
if (OperatingSystem.IsBrowser())
24+
{
25+
entryPoint = new WasmEntryPoint(runTestsCallback, assemblyName, filter);
26+
}
27+
if (entryPoint is null)
28+
{
29+
throw new InvalidOperationException("The XHarness runner library test runner is only supported on mobile+wasm platforms. Use the XUnitWrapperLibrary-based runner on desktop platforms.");
30+
}
31+
bool anyFailedTests = false;
32+
entryPoint.TestsCompleted += (o, e) => anyFailedTests = e.FailedTests > 0;
33+
await entryPoint.RunAsync();
34+
return anyFailedTests ? 1 : 0;
35+
}
36+
37+
sealed class AppleEntryPoint : iOSApplicationEntryPointBase
38+
{
39+
private Func<TestFilter?, TestSummary> _runTestsCallback;
40+
private string _assemblyName;
41+
private string? _methodNameToRun;
42+
43+
public AppleEntryPoint(IDevice device, Func<TestFilter?, TestSummary> runTestsCallback, string assemblyName, string? methodNameToRun)
44+
{
45+
Device = device;
46+
_runTestsCallback = runTestsCallback;
47+
_assemblyName = assemblyName;
48+
_methodNameToRun = methodNameToRun;
49+
}
50+
51+
protected override IDevice? Device { get; }
52+
protected override int? MaxParallelThreads => 1;
53+
protected override bool IsXunit => true;
54+
protected override TestRunner GetTestRunner(LogWriter logWriter)
55+
{
56+
var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName);
57+
if (_methodNameToRun is not null)
58+
{
59+
runner.SkipMethod(_methodNameToRun, isExcluded: false);
60+
}
61+
return runner;
62+
}
63+
64+
protected override IEnumerable<TestAssemblyInfo> GetTestAssemblies() => Array.Empty<TestAssemblyInfo>();
65+
protected override void TerminateWithSuccess() => Console.WriteLine("[TerminateWithSuccess]");
66+
}
67+
68+
sealed class AndroidEntryPoint : AndroidApplicationEntryPointBase
69+
{
70+
private Func<TestFilter?, TestSummary> _runTestsCallback;
71+
private string _assemblyName;
72+
private string? _methodNameToRun;
73+
74+
public AndroidEntryPoint(IDevice device, Func<TestFilter?, TestSummary> runTestsCallback, string assemblyName, string? methodNameToRun)
75+
{
76+
Device = device;
77+
_runTestsCallback = runTestsCallback;
78+
_assemblyName = assemblyName;
79+
_methodNameToRun = methodNameToRun;
80+
}
81+
82+
protected override IDevice? Device { get; }
83+
protected override int? MaxParallelThreads => 1;
84+
protected override bool IsXunit => true;
85+
protected override TestRunner GetTestRunner(LogWriter logWriter)
86+
{
87+
var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName);
88+
if (_methodNameToRun is not null)
89+
{
90+
runner.SkipMethod(_methodNameToRun, isExcluded: false);
91+
}
92+
return runner;
93+
}
94+
95+
public override string TestsResultsFinalPath
96+
{
97+
get
98+
{
99+
string? publicDir = Environment.GetEnvironmentVariable("DOCSDIR");
100+
if (string.IsNullOrEmpty(publicDir))
101+
throw new ArgumentException("DOCSDIR should not be empty");
102+
103+
return Path.Combine(publicDir, "testResults.xml");
104+
}
105+
}
106+
protected override IEnumerable<TestAssemblyInfo> GetTestAssemblies() => Array.Empty<TestAssemblyInfo>();
107+
108+
protected override void TerminateWithSuccess() {}
109+
110+
public override TextWriter? Logger => null;
111+
}
112+
113+
sealed class WasmEntryPoint : WasmApplicationEntryPointBase
114+
{
115+
private Func<TestFilter?, TestSummary> _runTestsCallback;
116+
private string _assemblyName;
117+
118+
private string? _methodNameToRun;
119+
120+
public WasmEntryPoint(Func<TestFilter?, TestSummary> runTestsCallback, string assemblyName, string? methodNameToRun)
121+
{
122+
_runTestsCallback = runTestsCallback;
123+
_assemblyName = assemblyName;
124+
_methodNameToRun = methodNameToRun;
125+
}
126+
protected override int? MaxParallelThreads => 1;
127+
protected override bool IsXunit => true;
128+
protected override TestRunner GetTestRunner(LogWriter logWriter)
129+
{
130+
var runner = new GeneratedTestRunner(logWriter, _runTestsCallback, _assemblyName);
131+
if (_methodNameToRun is not null)
132+
{
133+
runner.SkipMethod(_methodNameToRun, isExcluded: false);
134+
}
135+
return runner;
136+
}
137+
138+
protected override IEnumerable<TestAssemblyInfo> GetTestAssemblies() => Array.Empty<TestAssemblyInfo>();
139+
140+
}
141+
142+
class SimpleDevice : IDevice
143+
{
144+
public SimpleDevice(string assemblyName)
145+
{
146+
BundleIdentifier = "net.dot." + assemblyName;
147+
}
148+
149+
public string BundleIdentifier {get;}
150+
151+
public string? UniqueIdentifier { get; }
152+
153+
public string? Name { get; }
154+
155+
public string? Model { get; }
156+
157+
public string? SystemName { get; }
158+
159+
public string? SystemVersion { get; }
160+
161+
public string? Locale { get; }
162+
}
163+
}

‎src/tests/Common/XHarnessRunnerLibrary/XHarnessRunnerLibrary.csproj

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<OutputType>Library</OutputType>
4+
<CLRTestKind>BuildOnly</CLRTestKind>
5+
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
6+
<GenerateRunScript>false</GenerateRunScript>
7+
<TargetFramework>$(NetCoreAppToolCurrent)</TargetFramework>
8+
<EnableDefaultItems>true</EnableDefaultItems>
9+
<Nullable>enable</Nullable>
10+
<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
11+
</PropertyGroup>
12+
13+
<ItemGroup>
14+
<ProjectReference Include="../XUnitWrapperLibrary/XUnitWrapperLibrary.csproj" />
15+
<PackageReference Include="Microsoft.DotNet.XHarness.TestRunners.Common" Version="$(MicrosoftDotNetXHarnessTestRunnersCommonVersion)" />
16+
</ItemGroup>
17+
</Project>

‎src/tests/Common/XUnitWrapperGenerator/XUnitWrapperGenerator.cs

Lines changed: 37 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -126,11 +126,18 @@ public void Initialize(IncrementalGeneratorInitializationContext context)
126126
}
127127

128128
bool isMergedTestRunnerAssembly = configOptions.GlobalOptions.IsMergedTestRunnerAssembly();
129+
configOptions.GlobalOptions.TryGetValue("build_property.TargetOS", out string? targetOS);
129130

130-
// TODO: add error (maybe in MSBuild that referencing CoreLib directly from a merged test runner is not supported)
131131
if (isMergedTestRunnerAssembly)
132132
{
133-
context.AddSource("FullRunner.g.cs", GenerateFullTestRunner(methods, aliasMap, assemblyName));
133+
if (targetOS?.ToLowerInvariant() is "ios" or "iossimulator" or "tvos" or "tvossimulator" or "maccatalyst" or "android" or "browser")
134+
{
135+
context.AddSource("XHarnessRunner.g.cs", GenerateXHarnessTestRunner(methods, aliasMap, assemblyName));
136+
}
137+
else
138+
{
139+
context.AddSource("FullRunner.g.cs", GenerateFullTestRunner(methods, aliasMap, assemblyName));
140+
}
134141
}
135142
else
136143
{
@@ -162,6 +169,34 @@ private static string GenerateFullTestRunner(ImmutableArray<ITestInfo> testInfos
162169
return builder.ToString();
163170
}
164171

172+
private static string GenerateXHarnessTestRunner(ImmutableArray<ITestInfo> testInfos, ImmutableDictionary<string, string> aliasMap, string assemblyName)
173+
{
174+
// For simplicity, we'll use top-level statements for the generated Main method.
175+
StringBuilder builder = new();
176+
builder.AppendLine(string.Join("\n", aliasMap.Values.Where(alias => alias != "global").Select(alias => $"extern alias {alias};")));
177+
178+
179+
builder.AppendLine("try {");
180+
builder.AppendLine($@"return await XHarnessRunnerLibrary.RunnerEntryPoint.RunTests(RunTests, ""{assemblyName}"", args.Length != 0 ? args[0] : null);");
181+
builder.AppendLine("} catch(System.Exception ex) { System.Console.WriteLine(ex.ToString()); return 101; }");
182+
183+
builder.AppendLine("static XUnitWrapperLibrary.TestSummary RunTests(XUnitWrapperLibrary.TestFilter filter)");
184+
builder.AppendLine("{");
185+
builder.AppendLine("XUnitWrapperLibrary.TestSummary summary = new();");
186+
ITestReporterWrapper reporter = new WrapperLibraryTestSummaryReporting("summary", "filter");
187+
builder.AppendLine("System.Diagnostics.Stopwatch stopwatch = new();");
188+
189+
foreach (ITestInfo test in testInfos)
190+
{
191+
builder.AppendLine(test.GenerateTestExecution(reporter));
192+
}
193+
194+
builder.AppendLine("return summary;");
195+
builder.AppendLine("}");
196+
197+
return builder.ToString();
198+
}
199+
165200
private static string GenerateStandaloneSimpleTestRunner(ImmutableArray<ITestInfo> testInfos, ImmutableDictionary<string, string> aliasMap, string consoleType)
166201
{
167202
// For simplicity, we'll use top-level statements for the generated Main method.

0 commit comments

Comments
 (0)