Skip to content

Commit 9a590db

Browse files
halgarierri120
halgari
and
erri120
authored
Schema failsafes (#2227)
* Tests and start of the migration failsafes * Add db snapshot via lfs * Added the ability to load old rocksdb snapshots * add rocksdb.zip files to LFS * Provide some documentation and tests * Update src/NexusMods.DataModel.SchemaVersions/SchemaFingerprint.cs Co-authored-by: erri120 <[email protected]> * Update src/NexusMods.DataModel.SchemaVersions/NexusMods.DataModel.SchemaVersions.csproj Co-authored-by: erri120 <[email protected]> * Update src/NexusMods.DataModel.SchemaVersions/Migrations/UpsertFingerprint.cs Co-authored-by: erri120 <[email protected]> * Update tests/NexusMods.DataModel.SchemaVersions.Tests/NexusMods.DataModel.SchemaVersions.Tests.csproj Co-authored-by: erri120 <[email protected]> * Update src/NexusMods.DataModel.SchemaVersions/Migrations/UpsertFingerprint.cs Co-authored-by: erri120 <[email protected]> * Update tests/NexusMods.DataModel.SchemaVersions.Tests/NexusMods.DataModel.SchemaVersions.Tests.csproj Co-authored-by: erri120 <[email protected]> * Fix tests * Fix extra dependency I didn't intend to commit * Try and fix the tests on Mac * Missed two files somehow * Platform independent newlines * Switch to standardized newlines and ascii for fingerprints. They're already ASCII this just removes another layer of complexity * Update to build version that uses lfs * Fix how we store the `Created` date so it works with MacOS --------- Co-authored-by: erri120 <[email protected]>
1 parent 64971c1 commit 9a590db

23 files changed

+662
-16
lines changed

‎.gitattributes

+1-13
Original file line numberDiff line numberDiff line change
@@ -1,65 +1,53 @@
11
# Auto detect text files and perform LF normalization
22
* text=auto
3-
43
# Documents
54
*.md text diff=markdown
6-
75
# Graphics
86
*.png binary
97
*.jpg binary
108
*.jpeg binary
119
*.gif binary
1210
*.ico binary
1311
*.svg binary
14-
1512
# Scripts (Unix)
1613
*.bash text eol=lf
1714
*.sh text eol=lf
1815
*.zsh text eol=lf
19-
2016
# Scripts (Windows)
2117
*.bat text eol=crlf
2218
*.cmd text eol=crlf
2319
*.ps1 text eol=crlf
24-
2520
# Archives
2621
*.7z binary
2722
*.gz binary
2823
*.tar binary
2924
*.tgz binary
3025
*.zip binary
31-
3226
# Code files
3327
*.cs text diff=csharp
34-
3528
# Project files
3629
*.sln text eol=crlf
3730
*.csproj text eol=crlf
38-
3931
*.targets text eol=crlf
4032
*.filters text eol=crlf
4133
*.filters text eol=crlf
4234
*.vcxitems text eol=crlf
43-
4435
# Dynamic libraries
4536
*.so binary
4637
*.dylib binary
4738
*.dll binary
48-
4939
# Executables
5040
*.exe binary
5141
*.out binary
5242
*.app binary
53-
5443
# Text files where line endings should be preserved
5544
*.patch -text
56-
5745
# Exclude files from exporting
5846
.gitattributes export-ignore
5947
.gitignore export-ignore
6048
.gitkeep export-ignore
61-
6249
# Verify
6350
*.verified.txt text eol=lf working-tree-encoding=UTF-8
6451
*.verified.xml text eol=lf working-tree-encoding=UTF-8
6552
*.verified.json text eol=lf working-tree-encoding=UTF-8
53+
*.rocksdb.zip filter=lfs diff=lfs merge=lfs -text

‎.github/workflows/clean_environment_tests.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ jobs:
3030

3131
build-and-test:
3232
if: github.event_name == 'push' || github.event.pull_request.draft == false
33-
uses: Nexus-Mods/NexusMods.App.Meta/.github/workflows/dotnet-build-and-test-with-osx.yaml@9c4844439c53f8b8b9f64fb707c91b469238b15e
33+
uses: Nexus-Mods/NexusMods.App.Meta/.github/workflows/dotnet-build-and-test-with-osx.yaml@ae64a3be780a74e94b59ee463a413083013c8b0c
3434
with:
3535
extra-test-args: "--blame-hang-timeout 20m"
3636
test-filter: "RequiresNetworking!=True&FlakeyTest!=True"

‎Directory.Packages.props

+2-1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
4343
<PackageVersion Include="Polly.Core" Version="8.4.2" />
4444
<PackageVersion Include="Polly" Version="8.4.2" />
45+
<PackageVersion Include="Verify" Version="26.6.0" />
4546
<PackageVersion Include="ZstdSharp.Port" Version="0.8.1" />
4647
</ItemGroup>
4748
<ItemGroup>
@@ -136,4 +137,4 @@
136137
<PackageVersion Include="Splat.Microsoft.Extensions.Logging" Version="15.2.22" />
137138
<PackageVersion Include="TransparentValueObjects" Version="1.0.1" />
138139
</ItemGroup>
139-
</Project>
140+
</Project>

‎NexusMods.App.sln

+14
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Reso
268268
EndProject
269269
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Resources.Resilience", "src\Abstractions\NexusMods.Abstractions.Resources.Resilience\NexusMods.Abstractions.Resources.Resilience.csproj", "{04219A58-C99C-4C3B-A477-5E4B29D1F275}"
270270
EndProject
271+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.DataModel.SchemaVersions", "src\NexusMods.DataModel.SchemaVersions\NexusMods.DataModel.SchemaVersions.csproj", "{79E13AD1-187B-42F7-BDC3-EF8ABA308973}"
272+
EndProject
273+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.DataModel.SchemaVersions.Tests", "tests\NexusMods.DataModel.SchemaVersions.Tests\NexusMods.DataModel.SchemaVersions.Tests.csproj", "{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}"
274+
EndProject
271275
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Games.MountAndBlade2Bannerlord", "src\Games\NexusMods.Games.MountAndBlade2Bannerlord\NexusMods.Games.MountAndBlade2Bannerlord.csproj", "{8D7E82BB-2F8D-455A-AF12-C486D9EC3B77}"
272276
EndProject
273277
Global
@@ -700,6 +704,14 @@ Global
700704
{04219A58-C99C-4C3B-A477-5E4B29D1F275}.Debug|Any CPU.Build.0 = Debug|Any CPU
701705
{04219A58-C99C-4C3B-A477-5E4B29D1F275}.Release|Any CPU.ActiveCfg = Release|Any CPU
702706
{04219A58-C99C-4C3B-A477-5E4B29D1F275}.Release|Any CPU.Build.0 = Release|Any CPU
707+
{79E13AD1-187B-42F7-BDC3-EF8ABA308973}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
708+
{79E13AD1-187B-42F7-BDC3-EF8ABA308973}.Debug|Any CPU.Build.0 = Debug|Any CPU
709+
{79E13AD1-187B-42F7-BDC3-EF8ABA308973}.Release|Any CPU.ActiveCfg = Release|Any CPU
710+
{79E13AD1-187B-42F7-BDC3-EF8ABA308973}.Release|Any CPU.Build.0 = Release|Any CPU
711+
{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
712+
{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}.Debug|Any CPU.Build.0 = Debug|Any CPU
713+
{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}.Release|Any CPU.ActiveCfg = Release|Any CPU
714+
{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE}.Release|Any CPU.Build.0 = Release|Any CPU
703715
{8D7E82BB-2F8D-455A-AF12-C486D9EC3B77}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
704716
{8D7E82BB-2F8D-455A-AF12-C486D9EC3B77}.Debug|Any CPU.Build.0 = Debug|Any CPU
705717
{8D7E82BB-2F8D-455A-AF12-C486D9EC3B77}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -828,6 +840,8 @@ Global
828840
{BE8C17C4-E3B0-4D07-8CD0-0D15C3CCA9D5} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
829841
{D3BA5B5A-668A-443B-872C-3116CBB0BC0D} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
830842
{04219A58-C99C-4C3B-A477-5E4B29D1F275} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
843+
{79E13AD1-187B-42F7-BDC3-EF8ABA308973} = {E7BAE287-D505-4D6D-A090-665A64309B2D}
844+
{A5A2932D-B3EF-480B-BEBC-793F6FC90EDE} = {52AF9D62-7D5B-4AD0-BA12-86F2AA67428B}
831845
{8D7E82BB-2F8D-455A-AF12-C486D9EC3B77} = {70D38D24-79AE-4600-8E83-17F3C11BA81F}
832846
EndGlobalSection
833847
GlobalSection(ExtensibilityGlobals) = postSolution

‎src/NexusMods.App.UI/Pages/LoadoutPage/LoadoutPage.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public override ILoadoutViewModel CreateViewModel(LoadoutPageContext context)
4848
public override IEnumerable<PageDiscoveryDetails?> GetDiscoveryDetails(IWorkspaceContext workspaceContext)
4949
{
5050
if (workspaceContext is not LoadoutContext loadoutContext) yield break;
51-
51+
5252
yield return new PageDiscoveryDetails
5353
{
5454
SectionName = "Mods",

‎src/NexusMods.App/Program.cs

+7
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
using NexusMods.CrossPlatform;
1414
using NexusMods.CrossPlatform.Process;
1515
using NexusMods.DataModel;
16+
using NexusMods.DataModel.Migrations;
1617
using NexusMods.Paths;
1718
using NexusMods.ProxyConsole;
1819
using NexusMods.Settings;
@@ -59,9 +60,15 @@ public static int Main(string[] args)
5960
);
6061
var services = host.Services;
6162

63+
// Run the migrations
64+
var migration = services.GetRequiredService<MigrationService>();
65+
migration.Run().Wait();
66+
67+
6268
// Okay to do wait here, as we are in the main process thread.
6369
host.StartAsync().Wait(timeout: TimeSpan.FromMinutes(5));
6470

71+
6572
// Start the CLI server if we are the main process.
6673
var cliServer = services.GetService<CliServer>();
6774
cliServer?.StartCliServerAsync().Wait(timeout: TimeSpan.FromSeconds(5));

‎src/NexusMods.DataModel.SchemaVersions/IMigration.cs

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
using NexusMods.MnemonicDB.Abstractions;
2+
3+
namespace NexusMods.DataModel.Migrations;
4+
5+
/// <summary>
6+
/// A definition of a single data migration
7+
/// </summary>
8+
public interface IMigration
9+
{
10+
/// <summary>
11+
/// The name of the migration
12+
/// </summary>
13+
public string Name { get; }
14+
15+
/// <summary>
16+
/// A long description of the migration
17+
/// </summary>
18+
public string Description { get; }
19+
20+
/// <summary>
21+
/// A date for the migration's creation. Not used for anything other than sorting. Migrations
22+
/// will be run in order of this date.
23+
/// </summary>
24+
public DateTimeOffset CreatedAt { get; }
25+
26+
/// <summary>
27+
/// Returns true if the migration should run. This function should do any sort of querying and processing to make sure
28+
/// data is in the format expected by the migration.
29+
/// </summary>
30+
public bool ShouldRun(IDb db);
31+
32+
/// <summary>
33+
/// Runs the migration
34+
/// </summary>
35+
public void Migrate(IDb basis, ITransaction tx);
36+
}

‎src/NexusMods.DataModel.SchemaVersions/MigrationService.cs

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
using Microsoft.Extensions.Logging;
2+
using NexusMods.MnemonicDB.Abstractions;
3+
4+
namespace NexusMods.DataModel.Migrations;
5+
6+
/// <summary>
7+
/// Updates the state of a database and provides hooks for migrating schemas
8+
/// and transforming data between versions.
9+
/// </summary>
10+
public class MigrationService
11+
{
12+
private readonly ILogger<MigrationService> _logger;
13+
private readonly IConnection _connection;
14+
private readonly List<IMigration> _migrations;
15+
16+
public MigrationService(ILogger<MigrationService> logger, IConnection connection, IEnumerable<IMigration> migrations)
17+
{
18+
_logger = logger;
19+
_connection = connection;
20+
_migrations = migrations.OrderBy(m => m.CreatedAt).ToList();
21+
}
22+
23+
public async Task Run()
24+
{
25+
// Run all migrations, for now this interface works by handing a transaction to each migration, in the future we'll need
26+
// to add support for changing history of the datoms and not just the most recent state. But until we need such a migration
27+
// we'll go with this approach as it's simpler.
28+
foreach (var migration in _migrations)
29+
{
30+
var db = _connection.Db;
31+
if (!migration.ShouldRun(db))
32+
{
33+
_logger.LogInformation("Migration {Name} skipped", migration.Name);
34+
continue;
35+
}
36+
37+
_logger.LogInformation("Running migration {Name}", migration.Name);
38+
using var tx = _connection.BeginTransaction();
39+
migration.Migrate(db, tx);
40+
await tx.Commit();
41+
_logger.LogInformation("Migration {Name} completed", migration.Name);
42+
}
43+
}
44+
}

‎src/NexusMods.DataModel.SchemaVersions/Migrations/UpsertFingerprint.cs

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
using NexusMods.DataModel.SchemaVersions;
2+
using NexusMods.Hashing.xxHash3;
3+
using NexusMods.MnemonicDB.Abstractions;
4+
using NexusMods.MnemonicDB.Abstractions.ElementComparers;
5+
using NexusMods.MnemonicDB.Abstractions.ValueSerializers;
6+
7+
namespace NexusMods.DataModel.Migrations.Migrations;
8+
9+
public class UpsertFingerprint : IMigration
10+
{
11+
public string Name => "Upsert Fingerprint";
12+
public string Description => "Upserts the fingerprint of the database, creating it if it does not exist.";
13+
14+
/// <summary>
15+
/// Max value so it always runs last
16+
/// </summary>
17+
public DateTimeOffset CreatedAt => DateTimeOffset.MaxValue;
18+
19+
public bool ShouldRun(IDb db)
20+
{
21+
if (!db.AttributeCache.Has(SchemaVersion.Fingerprint.Id))
22+
return true;
23+
24+
var fingerprints = db.Datoms(SchemaVersion.Fingerprint);
25+
// No fingerprint, we need to create it
26+
if (fingerprints.Count == 0)
27+
return true;
28+
29+
var currentFingerprint = SchemaFingerprint.GenerateFingerprint(db);
30+
var dbFingerprint = Hash.From(UInt64Serializer.Read(fingerprints[0].ValueSpan));
31+
// Is the fingerprint up to date?
32+
return currentFingerprint != dbFingerprint;
33+
}
34+
35+
public void Migrate(IDb basis, ITransaction tx)
36+
{
37+
var eid = basis.Datoms(SchemaVersion.Fingerprint).Select(d => d.E)
38+
.FirstOrDefault(tx.TempId());
39+
40+
tx.Add(eid, SchemaVersion.Fingerprint, SchemaFingerprint.GenerateFingerprint(basis));
41+
}
42+
}

‎src/NexusMods.DataModel.SchemaVersions/NexusMods.DataModel.SchemaVersions.csproj

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<!-- NuGet Package Shared Details -->
3+
<Import Project="$([MSBuild]::GetPathOfFileAbove('NuGet.Build.props', '$(MSBuildThisFileDirectory)../'))" />
4+
5+
<ItemGroup>
6+
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
7+
<PackageReference Include="NexusMods.Hashing.xxHash3" />
8+
<PackageReference Include="NexusMods.MnemonicDB.Abstractions" />
9+
<PackageReference Include="NexusMods.MnemonicDB.SourceGenerator" PrivateAssets="all" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
10+
</ItemGroup>
11+
12+
<ItemGroup>
13+
<ProjectReference Include="..\Abstractions\NexusMods.Abstractions.MnemonicDB.Attributes\NexusMods.Abstractions.MnemonicDB.Attributes.csproj" />
14+
</ItemGroup>
15+
16+
</Project>

‎src/NexusMods.DataModel.SchemaVersions/SchemaFingerprint.cs

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
using System.Text;
2+
using NexusMods.Hashing.xxHash3;
3+
using NexusMods.MnemonicDB.Abstractions;
4+
5+
namespace NexusMods.DataModel.SchemaVersions;
6+
7+
/// <summary>
8+
/// Tools for generating a hash of all the attributes of a schema so that we can detect changes.
9+
/// </summary>
10+
public class SchemaFingerprint
11+
{
12+
public static Hash GenerateFingerprint(IDb db)
13+
{
14+
StringBuilder sb = new();
15+
var cache = db.AttributeCache;
16+
17+
18+
void AppendLine(string s)
19+
{
20+
// We want platform independent newlines.
21+
sb.Append(s);
22+
sb.Append("\n");
23+
}
24+
25+
foreach (var id in cache.AllAttributeIds.OrderBy(id => id.Id, StringComparer.Ordinal))
26+
{
27+
var aid = cache.GetAttributeId(id);
28+
AppendLine(id.ToString());
29+
AppendLine(cache.GetValueTag(aid).ToString());
30+
AppendLine(cache.IsIndexed(aid).ToString());
31+
AppendLine(cache.IsCardinalityMany(aid).ToString());
32+
AppendLine(cache.IsNoHistory(aid).ToString());
33+
AppendLine("--");
34+
}
35+
// Use ascii as the attribute names must be ascii and this makes data comparisons simpler.
36+
var bytes = Encoding.ASCII.GetBytes(sb.ToString());
37+
return bytes.xxHash3();
38+
}
39+
40+
}

‎src/NexusMods.DataModel.SchemaVersions/SchemaVersion.cs

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
using NexusMods.Abstractions.MnemonicDB.Attributes;
2+
using NexusMods.MnemonicDB.Abstractions.Models;
3+
4+
namespace NexusMods.DataModel.Migrations;
5+
6+
public partial class SchemaVersion : IModelDefinition
7+
{
8+
public const string Namespace = "NexusMods.DataModel.SchemaVersioning.SchemaVersionModel";
9+
10+
/// <summary>
11+
/// The current fingerprint of the database. This is used to detect when schema updates do not need to be performend,
12+
/// and the app can start without the rather expensive upgrade process.
13+
/// </summary>
14+
public static readonly HashAttribute Fingerprint = new(Namespace, "Fingerprint");
15+
}

‎src/NexusMods.DataModel.SchemaVersions/Services.cs

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
using Microsoft.Extensions.DependencyInjection;
2+
using NexusMods.DataModel.Migrations;
3+
using NexusMods.DataModel.Migrations.Migrations;
4+
5+
namespace NexusMods.DataModel.SchemaVersions;
6+
7+
public static class Services
8+
{
9+
public static IServiceCollection AddMigrations(this IServiceCollection services)
10+
{
11+
services.AddSchemaVersionModel();
12+
services.AddSingleton<IMigration, UpsertFingerprint>();
13+
services.AddSingleton<MigrationService>();
14+
return services;
15+
}
16+
17+
}

‎src/NexusMods.DataModel/NexusMods.DataModel.csproj

+1
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
<ProjectReference Include="..\Extensions\NexusMods.Extensions.DynamicData\NexusMods.Extensions.DynamicData.csproj" />
1818
<ProjectReference Include="..\Extensions\NexusMods.Extensions.Hashing\NexusMods.Extensions.Hashing.csproj" />
1919
<ProjectReference Include="..\NexusMods.App.GarbageCollection.DataModel\NexusMods.App.GarbageCollection.DataModel.csproj" />
20+
<ProjectReference Include="..\NexusMods.DataModel.SchemaVersions\NexusMods.DataModel.SchemaVersions.csproj" />
2021
<ProjectReference Include="..\NexusMods.ProxyConsole.Abstractions\NexusMods.ProxyConsole.Abstractions.csproj" />
2122
</ItemGroup>
2223

0 commit comments

Comments
 (0)