Game Hash DB Builder (#2527)

* Work on building the game hashes DB from indexed data

* Missing project file

* Latest changes to add Polly to the Steam/GOG clients

* Cleanup the verbs a bit

* Update the build command to hash the result

* Move some files around that are in the wrong place, make a impl library for FileHashes

* Implement a file hash service. Can download the hash database and update it.

* WIP using the hash database in the game indexing

* Fixing some tests failing from DI

* WIP, still trying to fix the tests

* Fix the tests I broke by using a struct 🤦

* Fix the endian issues

* Bit of cleanup and some comments

* Bit more cleanup

* Limit how often we spam the Github server for releases

* Clean up a lock we likely didn't need

* Switch to a manually created manifest.json file

* Handle PR feedback

* Don't always use the global chunk cache, only when needed

* Revert the change to the synchronizer, we're going to need to fix this in another way

* Fix the nuget references

* And fix the abstractions namespace as well

* Try and fix the issues in CI

* Put the file hashes in their own folder so we don't clobber everything.

Author: halgari

Abstractions/NexusMods.Abstractions.Steam/ISteamSession.cs

@@ -9,6 +9,11 @@ namespace NexusMods.Abstractions.Steam;
 /// </summary>
 public interface ISteamSession
 {
+    /// <summary>
+    /// Connect to the Steam session, performing any necessary setup.
+    /// </summary>
+    public Task Connect(CancellationToken token);
+    
     /// <summary>
     /// Get the product info for the specified app ID
     /// </summary>

Directory.Packages.props

@@ -16,8 +16,8 @@
     <PackageVersion Include="Microsoft.Extensions.TimeProvider.Testing" Version="9.0.0" />
     <PackageVersion Include="Nerdbank.FullDuplexStream" Version="1.1.12" />
     <PackageVersion Include="Nerdbank.Streams" Version="2.11.79" />
-    <PackageVersion Include="NexusMods.MnemonicDB" Version="0.9.98" />
-    <PackageVersion Include="NexusMods.MnemonicDB.Abstractions" Version="0.9.98" />
+    <PackageVersion Include="NexusMods.MnemonicDB" Version="0.9.99" />
+    <PackageVersion Include="NexusMods.MnemonicDB.Abstractions" Version="0.9.99" />
     <PackageVersion Include="NexusMods.Hashing.xxHash3.Paths" Version="3.0.3" />
     <PackageVersion Include="NexusMods.Hashing.xxHash3" Version="3.0.3" />
     <PackageVersion Include="NexusMods.Paths" Version="0.15.0" />
@@ -138,7 +138,7 @@
     <PackageVersion Include="Humanizer" Version="2.14.1" />
     <PackageVersion Include="ini-parser-netstandard" Version="2.5.2" />
     <PackageVersion Include="Mutagen.Bethesda.Skyrim" Version="0.44.0" />
-    <PackageVersion Include="NexusMods.MnemonicDB.SourceGenerator" Version="0.9.98" />
+    <PackageVersion Include="NexusMods.MnemonicDB.SourceGenerator" Version="0.9.99" />
     <PackageVersion Include="NLog.Extensions.Logging" Version="5.3.14" />
     <PackageVersion Include="OneOf" Version="3.0.271" />
     <PackageVersion Include="ReactiveUI.Fody" Version="19.5.41" />

NexusMods.App.sln

@@ -292,6 +292,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.GOG"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Logging", "src\Abstractions\NexusMods.Abstractions.Logging\NexusMods.Abstractions.Logging.csproj", "{9DE1C2AC-927A-4BC6-B2A1-7016902F8BAE}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Games.FileHashes", "src\Abstractions\NexusMods.Abstractions.Games.FileHashes\NexusMods.Abstractions.Games.FileHashes.csproj", "{E2DB1DF4-9934-4119-BA90-196FDDB0904A}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Games.FileHashes", "src\NexusMods.Games.FileHashes\NexusMods.Games.FileHashes.csproj", "{71D8CF63-A287-45AB-B251-676A54576C1D}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -770,6 +774,14 @@ Global
 		{9DE1C2AC-927A-4BC6-B2A1-7016902F8BAE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{9DE1C2AC-927A-4BC6-B2A1-7016902F8BAE}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{9DE1C2AC-927A-4BC6-B2A1-7016902F8BAE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E2DB1DF4-9934-4119-BA90-196FDDB0904A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E2DB1DF4-9934-4119-BA90-196FDDB0904A}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E2DB1DF4-9934-4119-BA90-196FDDB0904A}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E2DB1DF4-9934-4119-BA90-196FDDB0904A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{71D8CF63-A287-45AB-B251-676A54576C1D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{71D8CF63-A287-45AB-B251-676A54576C1D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{71D8CF63-A287-45AB-B251-676A54576C1D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{71D8CF63-A287-45AB-B251-676A54576C1D}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -906,6 +918,8 @@ Global
 		{F7FD18A7-2F00-4EB6-84FC-15C57326FDEB} = {D7E9D8F5-8AC8-4ADA-B219-C549084AD84C}
 		{03AC4F34-E69A-41E3-9F00-EE5A558D01B9} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
 		{9DE1C2AC-927A-4BC6-B2A1-7016902F8BAE} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
+		{E2DB1DF4-9934-4119-BA90-196FDDB0904A} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
+		{71D8CF63-A287-45AB-B251-676A54576C1D} = {70D38D24-79AE-4600-8E83-17F3C11BA81F}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {9F9F8352-34DD-42C0-8564-EE9AF34A3501}

src/Abstractions/NexusMods.Abstractions.Cli/RendererExtensions.cs

@@ -73,6 +73,16 @@ public static async ValueTask Text(this IRenderer renderer, string text)
         await renderer.RenderAsync(Renderable.Text(text));
     }
 
+    /// <summary>
+    /// Renders the given text to the renderer and follows it up with a newline
+    /// </summary>
+    /// <param name="renderer"></param>
+    /// <param name="text"></param>
+    public static async ValueTask TextLine(this IRenderer renderer, string text)
+    {
+        await renderer.RenderAsync(Renderable.Text(text + "\n"));
+    }
+    
     /// <summary>
     /// Starts a progress bar box in the renderer that will be stopped when the returned disposable is disposed
     /// </summary>
@@ -103,6 +113,17 @@ public static async ValueTask Text(this IRenderer renderer, string template, par
         await renderer.RenderAsync(Renderable.Text(template, args.Select(a => a.ToString()!).ToArray()));
     }
 
+    /// <summary>
+    /// Renders the text to the renderer with the given arguments and template, followed by a newline
+    /// </summary>
+    /// <param name="renderer"></param>
+    /// <param name="text"></param>
+    public static async ValueTask TextLine(this IRenderer renderer, string template, params object[] args)
+    {
+        // Todo: implement custom conversion and formatting for the arguments
+        await renderer.RenderAsync(Renderable.Text(template + "\n", args.Select(a => a.ToString()!).ToArray()));
+    }
+    
     /// <summary>
     /// Renders the text to the renderer with the given arguments and template
     /// </summary>
@@ -138,6 +159,18 @@ public static async Task Error(this IRenderer renderer, Exception ex, string tem
         await renderer.Text(template, args);
         await renderer.Text("Error: {0}", ex);
     }
+    
+    /// <summary>
+    /// Prints an error message to the renderer, and returns an error code
+    /// </summary>
+    /// <param name="renderer"></param>
+    /// <param name="ex"></param>
+    /// <param name="template"></param>
+    /// <param name="args"></param>
+    public static async Task Error(this IRenderer renderer, string template, params object[] args)
+    {
+        await renderer.Text(template, args);
+    }
 
     /// <summary>
     /// Creates a new progress task with the given text, the task will be deleted when the returned disposable is disposed
@@ -163,4 +196,20 @@ public static async IAsyncEnumerable<T> WithProgress<T>(this T[] items, IRendere
             yield return item;
         }
     }
+    
+    /// <summary>
+    /// Wraps the enumeration in a progress task that will update the progress bar as the items are enumerated
+    /// </summary>
+    public static async IAsyncEnumerable<T> WithProgress<T>(this IEnumerable<T> items, IRenderer renderer, string text)
+    {
+        var array = items.ToArray();
+        var increment = 1.0 / array.Length;
+        await using var task = await renderer.StartProgressTask(text);
+        
+        foreach (var item in array)
+        {
+            await task.IncrementProgress(increment);
+            yield return item;
+        }
+    }
 }

src/Abstractions/NexusMods.Abstractions.Games.FileHashes/Attributes/Crc32Attribute.cs

@@ -0,0 +1,22 @@
+using NexusMods.Abstractions.Hashes;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Attributes;
+
+/// <summary>
+/// An attribute for a CRC32 hash.
+/// </summary>
+public class Crc32Attribute(string ns, string name) : ScalarAttribute<Crc32, UInt32, UInt32Serializer>(ns, name)
+{
+    protected override uint ToLowLevel(Crc32 value)
+    {
+        return value.Value;
+    }
+
+    protected override Crc32 FromLowLevel(uint value, AttributeResolver resolver)
+    {
+        return Crc32.From(value);
+    }
+}

src/Abstractions/NexusMods.Abstractions.Games.FileHashes/Attributes/Gog/BuildIdAttribute.cs

@@ -0,0 +1,22 @@
+using NexusMods.Abstractions.GOG.Values;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Attributes.Gog;
+
+/// <summary>
+/// An attribute for storing GOG build IDs.
+/// </summary>
+public class BuildIdAttribute(string ns, string name) : ScalarAttribute<BuildId, ulong, UInt64Serializer>(ns, name)
+{
+    protected override ulong ToLowLevel(BuildId value)
+    {
+        return value.Value;
+    }
+
+    protected override BuildId FromLowLevel(ulong value, AttributeResolver resolver)
+    {
+        return BuildId.From(value);
+    }
+}

src/Abstractions/NexusMods.Abstractions.Games.FileHashes/Attributes/Gog/ProductIdAttribute.cs

@@ -0,0 +1,22 @@
+using NexusMods.Abstractions.GOG.Values;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Attributes.Gog;
+
+/// <summary>
+/// An attribute for storing GOG product IDs.
+/// </summary>
+public class ProductIdAttribute(string ns, string name) : ScalarAttribute<ProductId, ulong, UInt64Serializer>(ns, name)
+{
+    protected override ulong ToLowLevel(ProductId value)
+    {
+        return value.Value;
+    }
+
+    protected override ProductId FromLowLevel(ulong value, AttributeResolver resolver)
+    {
+        return ProductId.From(value);
+    }
+}

src/Abstractions/NexusMods.Abstractions.Games.FileHashes/Attributes/Md5Attribute.cs

@@ -0,0 +1,22 @@
+using NexusMods.Abstractions.Hashes;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Attributes;
+
+/// <summary>
+/// An attribute for storing MD5 hashes.
+/// </summary>
+public class Md5Attribute(string ns, string name) : ScalarAttribute<Md5, UInt128, UInt128Serializer>(ns, name)
+{
+    protected override UInt128 ToLowLevel(Md5 value)
+    {
+        return value.ToUInt128();
+    }
+
+    protected override Md5 FromLowLevel(UInt128 value, AttributeResolver resolver)
+    {
+        return Md5.FromUInt128(value);
+    }
+}

src/Abstractions/NexusMods.Abstractions.Games.FileHashes/Attributes/Sha1Attribute.cs

@@ -0,0 +1,22 @@
+using NexusMods.Abstractions.Hashes;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Attributes;
+
+/// <summary>
+/// An attribute for a SHA1 hash
+/// </summary>
+public class Sha1Attribute(string ns, string name) : ScalarAttribute<Sha1, Memory<byte>, BlobSerializer>(ns, name)
+{
+    protected override Memory<byte> ToLowLevel(Sha1 value)
+    {
+        return value.ToArray();
+    }
+
+    protected override Sha1 FromLowLevel(Memory<byte> value, AttributeResolver resolver)
+    {
+        return Sha1.From(value.Span);
+    }
+}

src/Abstractions/NexusMods.Abstractions.Games.FileHashes/Attributes/Steam/AppIdAttribute.cs

@@ -0,0 +1,22 @@
+using NexusMods.Abstractions.Steam.Values;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Attributes.Steam;
+
+/// <summary>
+/// An attribute for a Steam App ID.
+/// </summary>
+public class AppIdAttribute(string ns, string name) : ScalarAttribute<AppId, uint, UInt32Serializer>(ns, name) 
+{
+    protected override uint ToLowLevel(AppId value)
+    {
+        return value.Value;
+    }
+
+    protected override AppId FromLowLevel(uint value, AttributeResolver resolver)
+    {
+        return AppId.From(value);
+    }
+}

src/Abstractions/NexusMods.Abstractions.Games.FileHashes/Attributes/Steam/DepotIdAttribute.cs

@@ -0,0 +1,22 @@
+using NexusMods.Abstractions.Steam.Values;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Attributes.Steam;
+
+/// <summary>
+/// A <see cref="DepotId"/> attribute.
+/// </summary>
+public class DepotIdAttribute(string ns, string name) : ScalarAttribute<DepotId, uint, UInt32Serializer>(ns, name)
+{
+    protected override uint ToLowLevel(DepotId value)
+    {
+        return value.Value;
+    }
+
+    protected override DepotId FromLowLevel(uint value, AttributeResolver resolver)
+    {
+        return DepotId.From(value);
+    }
+}

src/Abstractions/NexusMods.Abstractions.Games.FileHashes/Attributes/Steam/ManifestIdAttribute.cs

@@ -0,0 +1,22 @@
+using NexusMods.Abstractions.Steam.Values;
+using NexusMods.MnemonicDB.Abstractions;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Attributes.Steam;
+
+/// <summary>
+/// A manifest ID attribute.
+/// </summary>
+public class ManifestIdAttribute(string ns, string name) : ScalarAttribute<ManifestId, ulong, UInt64Serializer>(ns, name)
+{
+    protected override ulong ToLowLevel(ManifestId value)
+    {
+        return value.Value;
+    }
+
+    protected override ManifestId FromLowLevel(ulong value, AttributeResolver resolver)
+    {
+        return ManifestId.From(value);
+    }
+}

src/Abstractions/NexusMods.Abstractions.Games.FileHashes/IFileHashesService.cs

@@ -0,0 +1,19 @@
+using NexusMods.MnemonicDB.Abstractions;
+
+namespace NexusMods.Abstractions.Games.FileHashes;
+
+/// <summary>
+/// Interface for the file hashes service, which provides a way to download and update the file hashes database
+/// </summary>
+public interface IFileHashesService
+{
+    /// <summary>
+    /// Force an update of the file hashes database
+    /// </summary>
+    public Task ForceUpdate();
+    
+    /// <summary>
+    /// Get the file hashes database, downloading it if necessary
+    /// </summary>
+    public ValueTask<IDb> GetFileHashesDb();
+}

src/Abstractions/NexusMods.Abstractions.Games.FileHashes/Models/GogBuild.cs

@@ -0,0 +1,49 @@
+using NexusMods.Abstractions.Games.FileHashes.Attributes.Gog;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.Models;
+using OperatingSystem = NexusMods.Abstractions.Games.FileHashes.Values.OperatingSystem;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Models;
+
+/// <summary>
+/// Metadata for a GOG build.
+/// </summary>
+public partial class GogBuild : IModelDefinition
+{
+    private const string Namespace = "NexusMods.Abstractions.Games.FileHashes.GogBuild";
+    
+    /// <summary>
+    /// The GOG build ID.
+    /// </summary>
+    public static readonly BuildIdAttribute BuildId = new(Namespace, nameof(BuildId)) { IsIndexed = true };
+    
+    /// <summary>
+    /// The GOG product ID.
+    /// </summary>
+    public static readonly ProductIdAttribute ProductId = new(Namespace, nameof(ProductId)) { IsIndexed = true };
+    
+    /// <summary>
+    /// The Operating System the build is for.
+    /// </summary>
+    public static readonly EnumByteAttribute<OperatingSystem> OperatingSystem = new(Namespace, nameof(OperatingSystem));
+    
+    /// <summary>
+    /// The version string of the GOG build.
+    /// </summary>
+    public static readonly StringAttribute VersionName = new(Namespace, nameof(Version));
+    
+    /// <summary>
+    /// Various tags for the build
+    /// </summary>
+    public static readonly StringsAttribute Tags = new(Namespace, nameof(Tags));
+    
+    /// <summary>
+    /// True if the build is public, false if it is private.
+    /// </summary>
+    public static readonly BooleanAttribute Public = new(Namespace, nameof(Public));
+    
+    /// <summary>
+    /// The files in the GOG build.
+    /// </summary>
+    public static readonly ReferencesAttribute<PathHashRelation> Files = new(Namespace, nameof(Files));
+}