Skip to content

Commit ca81f4d

Browse files
halgarierri120Al12rs
authored
GOG file hashing (#2407)
* Basic structure of the projects, can login via a test, and stay logged in * Implemented the rest of the Steam flow for getting manifests. Introduced a .Hashes abstraction library for hash value types. Moved Chunked streams into the IO abstractions library * Boom, can stream files off of Steam * Can download files, serialize manifests, all that good stuff * Fix the chunked stream so that it can use arbitrarily sized chunks * Delete the old Store project * Disable the test that has to be run with human interaction * Few more comments and cleanup * WIP, can mostly log into GOG at this point * Can log into GOG and get product info, the format for the files themselves is rather whack, but we can make it work * Update Abstractions/NexusMods.Abstractions.Steam/DTOs/ProductInfo.cs Co-authored-by: erri120 <[email protected]> * Update src/Abstractions/NexusMods.Abstractions.Hashes/Crc32.cs Co-authored-by: erri120 <[email protected]> * Update src/Networking/NexusMods.Networking.Steam/NexusMods.Networking.Steam.csproj Co-authored-by: erri120 <[email protected]> * Update src/Networking/NexusMods.Networking.Steam/Session.cs Co-authored-by: erri120 <[email protected]> * Update Abstractions/NexusMods.Abstractions.Steam/DTOs/Manifest.cs Co-authored-by: erri120 <[email protected]> * Update src/Networking/NexusMods.Networking.Steam/ProductInfoParser.cs Co-authored-by: erri120 <[email protected]> * Update src/Abstractions/NexusMods.Abstractions.Hashes/MultiHasher.cs Co-authored-by: erri120 <[email protected]> * Update src/Networking/NexusMods.Networking.Steam/Session.cs Co-authored-by: erri120 <[email protected]> * WIP getting access to files * `sizeof(ulong)` vs `size(long)` * Update doc strings to better explain the value sizes * Downloading and basic indexing of GOG data * Can hash depots and export data, and it's not horrible performance * Update src/Networking/NexusMods.Networking.Steam/ManifestParser.cs Co-authored-by: erri120 <[email protected]> * Export all OSes * Update src/Abstractions/NexusMods.Abstractions.Hashes/HashJsonConverter.cs Co-authored-by: erri120 <[email protected]> * Fix the .Steam project * fix compile errors introduced during code review * Upgrade deps, fix several broken tests on `main` * Bit of code cleanup * Add some doc strings * Fix tests so we can get an idea of build sizes * try a change to fix builds on osx * include osx explicitly * More config changes * Include ARM64 binaries * Fix verify tests * Missed a test file * Switch around a few config things, just getting unblocked for now * Fix the endianness of the hash converter * WIP switch over to actual oauth flow * WIP working login * Today's work, need some feedback from GOG * WIP console progress * Finish integrating the gog login and clean up the indexing code a bit * Bit of documentation * Delete references to CEF/DotNetBrowser * Bit more cleanup * Update src/Networking/NexusMods.Networking.GOG/Client.cs Co-authored-by: Al <[email protected]> * Handle some PR feedback * Include the rest of the comments * Bit more comments * Cleanup some code in SpectreRenderer.cs --------- Co-authored-by: erri120 <[email protected]> Co-authored-by: Al <[email protected]>
1 parent 0b5f2be commit ca81f4d

File tree

59 files changed

+1789
-41
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+1789
-41
lines changed

Abstractions/NexusMods.Abstractions.Steam/NexusMods.Abstractions.Steam.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,6 @@
77
</ItemGroup>
88

99
<ItemGroup>
10-
<ProjectReference Include="..\..\src\Abstractions\NexusMods.Abstractions.Hashes\NexusMods.Abstractions.Hashes.csproj" />
10+
<ProjectReference Include="..\..\src\Abstractions\NexusMods.Abstractions.Hashes\NexusMods.Abstractions.Hashes.csproj" />
1111
</ItemGroup>
1212
</Project>

Directory.Packages.props

+3-1
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
<PackageVersion Include="Avalonia.Labs.Panels" Version="11.2.0" />
55
<PackageVersion Include="Avalonia.Skia" Version="11.2.2" />
66
<PackageVersion Include="AvaloniaEdit.TextMate" Version="11.1.0" />
7+
<PackageVersion Include="Jitbit.FastCache" Version="1.1.0" />
78
<PackageVersion Include="K4os.Compression.LZ4" Version="1.3.8" />
89
<PackageVersion Include="Bannerlord.ModuleManager" Version="6.0.246" />
910
<PackageVersion Include="BsDiff" Version="1.1.0" />
@@ -37,6 +38,7 @@
3738
<PackageVersion Include="R3" Version="1.2.9" />
3839
<PackageVersion Include="R3Extensions.Avalonia" Version="1.2.9" />
3940
<PackageVersion Include="ReactiveUI" Version="20.1.63" />
41+
<PackageVersion Include="SmartFormat" Version="3.5.1" />
4042
<PackageVersion Include="Spectre.Console.Testing" Version="0.49.1" />
4143
<PackageVersion Include="SteamKit2" Version="3.0.0" />
4244
<PackageVersion Include="StrawberryShake.Server" Version="14.1.0" />
@@ -144,4 +146,4 @@
144146
<PackageVersion Include="Splat.Microsoft.Extensions.Logging" Version="15.2.22" />
145147
<PackageVersion Include="TransparentValueObjects" Version="1.0.2" />
146148
</ItemGroup>
147-
</Project>
149+
</Project>

NexusMods.App.sln

+14
Original file line numberDiff line numberDiff line change
@@ -286,6 +286,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Networking.Steam.
286286
EndProject
287287
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Hashes", "src\Abstractions\NexusMods.Abstractions.Hashes\NexusMods.Abstractions.Hashes.csproj", "{AF703852-D7B0-4BAD-8C75-B6046C6F0490}"
288288
EndProject
289+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Networking.GOG", "src\Networking\NexusMods.Networking.GOG\NexusMods.Networking.GOG.csproj", "{F7FD18A7-2F00-4EB6-84FC-15C57326FDEB}"
290+
EndProject
291+
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.GOG", "src\Abstractions\NexusMods.Abstractions.GOG\NexusMods.Abstractions.GOG.csproj", "{03AC4F34-E69A-41E3-9F00-EE5A558D01B9}"
292+
EndProject
289293
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NexusMods.Abstractions.Logging", "src\Abstractions\NexusMods.Abstractions.Logging\NexusMods.Abstractions.Logging.csproj", "{9DE1C2AC-927A-4BC6-B2A1-7016902F8BAE}"
290294
EndProject
291295
Global
@@ -754,6 +758,14 @@ Global
754758
{AF703852-D7B0-4BAD-8C75-B6046C6F0490}.Debug|Any CPU.Build.0 = Debug|Any CPU
755759
{AF703852-D7B0-4BAD-8C75-B6046C6F0490}.Release|Any CPU.ActiveCfg = Release|Any CPU
756760
{AF703852-D7B0-4BAD-8C75-B6046C6F0490}.Release|Any CPU.Build.0 = Release|Any CPU
761+
{F7FD18A7-2F00-4EB6-84FC-15C57326FDEB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
762+
{F7FD18A7-2F00-4EB6-84FC-15C57326FDEB}.Debug|Any CPU.Build.0 = Debug|Any CPU
763+
{F7FD18A7-2F00-4EB6-84FC-15C57326FDEB}.Release|Any CPU.ActiveCfg = Release|Any CPU
764+
{F7FD18A7-2F00-4EB6-84FC-15C57326FDEB}.Release|Any CPU.Build.0 = Release|Any CPU
765+
{03AC4F34-E69A-41E3-9F00-EE5A558D01B9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
766+
{03AC4F34-E69A-41E3-9F00-EE5A558D01B9}.Debug|Any CPU.Build.0 = Debug|Any CPU
767+
{03AC4F34-E69A-41E3-9F00-EE5A558D01B9}.Release|Any CPU.ActiveCfg = Release|Any CPU
768+
{03AC4F34-E69A-41E3-9F00-EE5A558D01B9}.Release|Any CPU.Build.0 = Release|Any CPU
757769
{9DE1C2AC-927A-4BC6-B2A1-7016902F8BAE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
758770
{9DE1C2AC-927A-4BC6-B2A1-7016902F8BAE}.Debug|Any CPU.Build.0 = Debug|Any CPU
759771
{9DE1C2AC-927A-4BC6-B2A1-7016902F8BAE}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -891,6 +903,8 @@ Global
891903
{24457AAA-8954-4BD6-8EB5-168EAC6EFB1B} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
892904
{17023DB9-8E31-4397-B3E1-141149987865} = {897C4198-884F-448A-B0B0-C2A6D971EAE0}
893905
{AF703852-D7B0-4BAD-8C75-B6046C6F0490} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
906+
{F7FD18A7-2F00-4EB6-84FC-15C57326FDEB} = {D7E9D8F5-8AC8-4ADA-B219-C549084AD84C}
907+
{03AC4F34-E69A-41E3-9F00-EE5A558D01B9} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
894908
{9DE1C2AC-927A-4BC6-B2A1-7016902F8BAE} = {0CB73565-1207-4A56-A79F-6A8E9BBD795C}
895909
EndGlobalSection
896910
GlobalSection(ExtensibilityGlobals) = postSolution

NexusMods.App.sln.DotSettings

+1
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/WRAP_BEFORE_INVOCATION_RPAR/@EntryValue">True</s:Boolean>
3737
<s:Int64 x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/WRAP_LIMIT/@EntryValue">200</s:Int64>
3838
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=API/@EntryIndexedValue">API</s:String>
39+
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=CDN/@EntryIndexedValue">CDN</s:String>
3940
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=DI/@EntryIndexedValue">DI</s:String>
4041
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=DTO/@EntryIndexedValue">DTO</s:String>
4142
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=EA/@EntryIndexedValue">EA</s:String>
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
using System.Diagnostics;
2+
using NexusMods.ProxyConsole.Abstractions;
3+
using NexusMods.ProxyConsole.Abstractions.Implementations;
4+
5+
namespace NexusMods.Abstractions.Cli;
6+
7+
/// <summary>
8+
/// A wrapper for progress task communication
9+
/// </summary>
10+
public class ProgressTask : IAsyncDisposable
11+
{
12+
private readonly IRenderer _renderer;
13+
private readonly Guid _taskId;
14+
private double _progress = 0;
15+
private readonly double? _maxValue;
16+
17+
internal ProgressTask(IRenderer renderer, Guid taskId, double? maxValue)
18+
{
19+
_renderer = renderer;
20+
_taskId = taskId;
21+
_maxValue = maxValue;
22+
}
23+
24+
/// <summary>
25+
/// Deletes the progress task
26+
/// </summary>
27+
public async ValueTask DisposeAsync()
28+
{
29+
await _renderer.RenderAsync(new DeleteProgressTask { TaskId = _taskId });
30+
}
31+
32+
/// <summary>
33+
/// Sets the progress of this task to the given value, in a range of 0 to 1
34+
/// </summary>
35+
public async Task SetProgress(double progress)
36+
{
37+
var increment = progress - _progress;
38+
_progress = progress;
39+
await _renderer.RenderAsync(new UpdateTask { TaskId = _taskId, IncrementProgressBy = increment });
40+
}
41+
42+
/// <summary>
43+
/// Increments the progress of this task by the given value, in a range of 0 to 1
44+
/// </summary>
45+
/// <param name="increment"></param>
46+
public async Task IncrementProgress(double increment)
47+
{
48+
_progress += increment;
49+
await _renderer.RenderAsync(new UpdateTask { TaskId = _taskId, IncrementProgressBy = increment });
50+
}
51+
52+
/// <summary>
53+
/// Increments the total "value" of this task by the given increment, the progress is then calculated as a percentage of the total value
54+
/// </summary>
55+
public async Task Increment(double increment)
56+
{
57+
Debug.Assert(_maxValue.HasValue);
58+
var progress = (double)increment / _maxValue!.Value;
59+
await IncrementProgress(progress);
60+
}
61+
}

src/Abstractions/NexusMods.Abstractions.Cli/RendererExtensions.cs

+44
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,25 @@ public static async ValueTask Text(this IRenderer renderer, string text)
7272
{
7373
await renderer.RenderAsync(Renderable.Text(text));
7474
}
75+
76+
/// <summary>
77+
/// Starts a progress bar box in the renderer that will be stopped when the returned disposable is disposed
78+
/// </summary>
79+
public static async Task<IAsyncDisposable> WithProgress(this IRenderer renderer)
80+
{
81+
await renderer.RenderAsync(new StartProgress());
82+
83+
return new DisposableProgress(renderer);
84+
}
85+
86+
private class DisposableProgress(IRenderer renderer) : IAsyncDisposable
87+
{
88+
public async ValueTask DisposeAsync()
89+
{
90+
await renderer.RenderAsync(new StopProgress());
91+
}
92+
}
93+
7594

7695
/// <summary>
7796
/// Renders the text to the renderer with the given arguments and template
@@ -119,4 +138,29 @@ public static async Task Error(this IRenderer renderer, Exception ex, string tem
119138
await renderer.Text(template, args);
120139
await renderer.Text("Error: {0}", ex);
121140
}
141+
142+
/// <summary>
143+
/// Creates a new progress task with the given text, the task will be deleted when the returned disposable is disposed
144+
/// </summary>
145+
public static async ValueTask<ProgressTask> StartProgressTask(this IRenderer renderer, string text, double? maxValue = null)
146+
{
147+
var taskId = Guid.NewGuid();
148+
await renderer.RenderAsync(new CreateProgressTask() { TaskId = taskId, Text = text });
149+
return new ProgressTask(renderer, taskId, maxValue);
150+
}
151+
152+
/// <summary>
153+
/// Wraps the enumeration in a progress task that will update the progress bar as the items are enumerated
154+
/// </summary>
155+
public static async IAsyncEnumerable<T> WithProgress<T>(this T[] items, IRenderer renderer, string text)
156+
{
157+
var increment = 1.0 / items.Length;
158+
await using var task = await renderer.StartProgressTask(text);
159+
160+
foreach (var item in items)
161+
{
162+
await task.IncrementProgress(increment);
163+
yield return item;
164+
}
165+
}
122166
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
using System.Text.Json.Serialization;
2+
using JetBrains.Annotations;
3+
using NexusMods.Abstractions.GOG.JsonConverters;
4+
using NexusMods.Abstractions.GOG.Values;
5+
6+
namespace NexusMods.Abstractions.GOG.DTOs;
7+
8+
/// <summary>
9+
/// Information about a build, which is a collection of files and the chunks that make up those files. This may sound confusing, but the idea
10+
/// is that "build" and to some extent "depot" are both metadata concepts. The Build is the information about what a collection of files are tagged
11+
/// as (like Cyberpunk 1.5) and the depot is metadata about the actual files that are stored on the CDN. There is a 1:1 relationship between depots and
12+
/// builds. The files in the depot are then stored in one of many CDNs, these CDN links are known as "SecureLinks". There may be many secure links (mirrors)
13+
/// for a given depot and build.
14+
/// </summary>
15+
[UsedImplicitly]
16+
public class Build
17+
{
18+
/// <summary>
19+
/// The unique ID of the build.
20+
/// </summary>
21+
[JsonPropertyName("build_id")]
22+
public required BuildId BuildId { get; init; }
23+
24+
/// <summary>
25+
/// The product ID of the build.
26+
/// </summary>
27+
[JsonPropertyName("product_id")]
28+
public required ProductId ProductId { get; init; }
29+
30+
/// <summary>
31+
/// The OS of the build.
32+
/// </summary>
33+
[JsonPropertyName("os")]
34+
public required string OS { get; init; }
35+
36+
/// <summary>
37+
/// The version of the build.
38+
/// </summary>
39+
[JsonPropertyName("version_name")]
40+
public required string VersionName { get; init; }
41+
42+
/// <summary>
43+
/// Various tags for the build (not sure what these may contain).
44+
/// </summary>
45+
[JsonPropertyName("tags")]
46+
public required string[] Tags { get; init; }
47+
48+
/// <summary>
49+
/// True if the build is public, false if it is private.
50+
/// </summary>
51+
[JsonPropertyName("public")]
52+
public bool Public { get; init; }
53+
54+
/// <summary>
55+
/// The date the build was published.
56+
/// </summary>
57+
[JsonConverter(typeof(GOGDateTimeOffsetConverter))]
58+
[JsonPropertyName("date_published")]
59+
public DateTimeOffset DatePublished { get; init; }
60+
61+
/// <summary>
62+
/// The generation of the build data (should be 2).
63+
/// </summary>
64+
[JsonPropertyName("generation")]
65+
public int Generation { get; init; }
66+
67+
/// <summary>
68+
/// A link to the build's details
69+
/// </summary>
70+
[JsonPropertyName("link")]
71+
public required Uri? Link { get; init; }
72+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
using System.Text.Json.Serialization;
2+
using JetBrains.Annotations;
3+
using NexusMods.Abstractions.GOG.Values;
4+
using NexusMods.Paths;
5+
6+
namespace NexusMods.Abstractions.GOG.DTOs;
7+
8+
/// <summary>
9+
/// Information about a build, which is a collection of depots.
10+
/// </summary>
11+
[UsedImplicitly]
12+
public class BuildDetails
13+
{
14+
[JsonPropertyName("baseProductId")]
15+
public required ProductId BaseProductId { get; init; }
16+
17+
[JsonPropertyName("buildId")]
18+
public required BuildId BuildId { get; init; }
19+
20+
[JsonPropertyName("dependencies")]
21+
public string[] Dependencies { get; init; } = [];
22+
23+
[JsonPropertyName("depots")]
24+
public required BuildDetailsDepot[] Depots { get; init; }
25+
}
26+
27+
/// <summary>
28+
/// Details about a depot in a build.
29+
/// </summary>
30+
[UsedImplicitly]
31+
public class BuildDetailsDepot
32+
{
33+
/// <summary>
34+
/// The size of the depot when compressed.
35+
/// </summary>
36+
[JsonPropertyName("compressedSize")]
37+
public required Size CompressedSize { get; init; }
38+
39+
/// <summary>
40+
/// The languages supported by the depot.
41+
/// </summary>
42+
[JsonPropertyName("languages")]
43+
public string[] Languages { get; init; } = [];
44+
45+
/// <summary>
46+
/// The unique ID of the manifest for this depot.
47+
/// </summary>
48+
[JsonPropertyName("manifest")]
49+
public required string Manifest { get; init; }
50+
51+
/// <summary>
52+
/// The OS bitness (32, 64) supported by the depot.
53+
/// </summary>
54+
[JsonPropertyName("osBitness")]
55+
public int[] OSBitness { get; init; } = [];
56+
57+
/// <summary>
58+
/// The product ID of the depot.
59+
/// </summary>
60+
[JsonPropertyName("productId")]
61+
public required ProductId ProductId { get; init; }
62+
63+
/// <summary>
64+
/// The size of the depot when uncompressed.
65+
/// </summary>
66+
[JsonPropertyName("size")]
67+
public required Size Size { get; init; }
68+
}

0 commit comments

Comments
 (0)