Add Tests for provided solutions

.github/workflows/publish-site.yml

@@ -7,7 +7,6 @@ on:
 
 jobs:
   publish:
-    name: build, pack & publish
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -21,7 +20,9 @@ jobs:
         run: dotnet restore
       - name: Pre-build solution
         run: dotnet build LearnJsonEverything.sln -c Release --no-restore
-      - name: Build
+      - name: Test
+        run: dotnet test LearnJsonEverything.sln -c Release --no-restore
+      - name: Publish
         run: dotnet publish LearnJsonEverything/LearnJsonEverything.csproj -c Release --no-restore -o bin
       - name: Add .nojekyll file
         run: touch bin/wwwroot/.nojekyll

.github/workflows/test.yml

@@ -0,0 +1,19 @@
+name: Verify solutions
+on:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: restore submodules
+        run: git submodule update --init
+      - name: Setup .NET Core
+        uses: actions/setup-dotnet@v2
+        with:
+          dotnet-version: 8.0.x
+      - name: Install dependencies
+        run: dotnet restore
+      - name: Test
+        run: dotnet test LearnJsonEverything.sln -c Release --no-restore

LearnJsonEverything.Tests/LearnJsonEverything.Tests.csproj

@@ -0,0 +1,34 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net8.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+    <IsTestProject>true</IsTestProject>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <None Include="..\LearnJsonEverything\wwwroot\data\lessons\*.yaml">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="coverlet.collector" Version="6.0.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.8.0" />
+    <PackageReference Include="NUnit" Version="3.14.0" />
+    <PackageReference Include="NUnit.Analyzers" Version="3.9.0" />
+    <PackageReference Include="NUnit3TestAdapter" Version="4.5.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\LearnJsonEverything\LearnJsonEverything.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Using Include="NUnit.Framework" />
+  </ItemGroup>
+
+</Project>

LearnJsonEverything.Tests/ProvidedSolutionTests.cs

@@ -0,0 +1,54 @@
+using System.Text.Json;
+using LearnJsonEverything.Services;
+using LearnJsonEverything.Services.Runners;
+using Yaml2JsonNode;
+
+namespace LearnJsonEverything.Tests;
+
+public class ProvidedSolutionTests
+{
+	private static JsonSerializerOptions SerializerOptions =
+		new()
+		{
+			PropertyNameCaseInsensitive = true
+		};
+
+	[OneTimeSetUp]
+	public void Setup()
+	{
+		CompilationHelpers.TestOnly_SetReferences(ReferenceLoader.Load());
+	}
+
+	private static LessonPlan LoadLessonPlan(string filename)
+	{
+		var yamlText = File.ReadAllText(filename);
+		var yaml = YamlSerializer.Parse(yamlText);
+		var json = yaml.First().ToJsonNode();
+		return json.Deserialize<LessonPlan>(SerializerOptions)!;
+	}
+
+	public static IEnumerable<TestCaseData> SchemaLessons
+	{
+		get
+		{
+			var lessonPlan = LoadLessonPlan("schema.yaml");
+			return lessonPlan.Select(x =>
+			{
+				x.UserCode = x.Solution;
+				return new TestCaseData(x) { TestName = x.Title };
+			});
+		}
+	}
+
+	[TestCaseSource(nameof(SchemaLessons))]
+	public void Schema(LessonData lesson)
+	{
+		var results = SchemaRunner.Run(lesson);
+
+		foreach (var result in results)
+		{
+			Console.WriteLine(result);
+			Assert.That(result, Does.StartWith(Iconography.SuccessIcon));
+		}
+	}
+}

LearnJsonEverything.Tests/ReferenceLoader.cs

@@ -0,0 +1,43 @@
+using System.Text.Json.Nodes;
+using Json.Schema;
+using Microsoft.CodeAnalysis;
+using Yaml2JsonNode;
+
+namespace LearnJsonEverything.Tests;
+
+public static class ReferenceLoader
+{
+	private class NullRunner : ILessonRunner<int>
+	{
+		public int Run(JsonObject context) => 0;
+	}
+
+	static ReferenceLoader()
+	{
+		// force some assemblies to load
+		SchemaRegistry.Global.Fetch = null!;
+		_ = YamlSerializer.Parse(string.Empty);
+		_ = new NullRunner();
+	}
+
+	public static MetadataReference[] Load()
+	{
+		var refs = AppDomain.CurrentDomain.GetAssemblies();
+		var assemblies = refs
+			.Where(x => !x.IsDynamic)
+			.OrderBy(x => x.FullName)
+			.ToArray();
+
+		var references = new MetadataReference[assemblies.Length];
+		int i = 0;
+		foreach (var assembly in assemblies)
+		{
+			Console.WriteLine($"Loading {assembly.FullName}...");
+			references[i] = MetadataReference.CreateFromFile(assembly.Location);
+			i++;
+		}
+
+		return references;
+	}
+
+}

LearnJsonEverything.sln

@@ -5,7 +5,9 @@ VisualStudioVersion = 17.9.34622.214
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LearnJsonEverything", "LearnJsonEverything\LearnJsonEverything.csproj", "{82FEC514-66E5-4034-9845-7C66B753D6FD}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LearnJsonEverything.Template", "LearnJsonEverything.Template\LearnJsonEverything.Template.csproj", "{B85D9C1F-3C5B-40E5-905B-9D5461E7E076}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LearnJsonEverything.Template", "LearnJsonEverything.Template\LearnJsonEverything.Template.csproj", "{B85D9C1F-3C5B-40E5-905B-9D5461E7E076}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LearnJsonEverything.Tests", "LearnJsonEverything.Tests\LearnJsonEverything.Tests.csproj", "{F01F40CE-A765-4DAB-85C2-39C391BAC00D}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -21,6 +23,10 @@ Global
 		{B85D9C1F-3C5B-40E5-905B-9D5461E7E076}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{B85D9C1F-3C5B-40E5-905B-9D5461E7E076}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{B85D9C1F-3C5B-40E5-905B-9D5461E7E076}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F01F40CE-A765-4DAB-85C2-39C391BAC00D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F01F40CE-A765-4DAB-85C2-39C391BAC00D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F01F40CE-A765-4DAB-85C2-39C391BAC00D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F01F40CE-A765-4DAB-85C2-39C391BAC00D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

LearnJsonEverything/Services/CompilationHelpers.cs

@@ -19,6 +19,8 @@ public static class CompilationHelpers
 		"Yaml2JsonNode",
 	];
 
+	public static void TestOnly_SetReferences(MetadataReference[] references) => _references = references;
+
 	public static async Task<MetadataReference[]?> LoadAssemblyReferences(HttpClient client)
 	{
 		if (_references is null)

LearnJsonEverything/Services/LessonPlan.cs

@@ -1,6 +1,5 @@
 using System.Collections;
 using System.Text.Json;
-using System.Text.Json.Nodes;
 using System.Text.Json.Serialization;
 
 namespace LearnJsonEverything.Services;

LearnJsonEverything/wwwroot/data/lessons/schema.yaml

@@ -200,7 +200,7 @@
         }
     }
   solution: |-
-    builder.Type(SchemaValueType.String)
+    builder.Type(SchemaValueType.Number)
         .ExclusiveMinimum(0)
         .Maximum(10);
   tests: