Remove dependency on default .NET extraction process on program startup

Complete refactoring the integration of native dependencies to move the remaining ones that triggered the default .NET extraction process on program startup.

In earlier versions, on MacOS environments, we observed crashes with error messages like the following:

```
Failure processing application bundle. Failed to determine location for extracting embedded files. DOTNET_BUNDLE_EXTRACT_BASE_DIR is not set, and a read-write cache directory couldn't be created.
```

The changes in this commit complete the refactoring around native dependencies to avoid these crashes.

+ Adapt consumers of LibGit2Sharp to invoke the setup of native dependencies if necessary.
+ Refactor moving common functionality around the setup of native dependencies in a shared module.
+ Adapt the build automation to disable the extraction and adapt the collection of files for releases to avoid noise in the downloads for users.
+ Expand the testing automation to adapt to the recent observations of problems with integrating native dependencies: Add the `self-test` command on the command-line interface to run tests for functionality that depends on native dependencies, such as libgit2 and ClearScript.V8. Expand the automated checks to integrate these new tests with the same builds published with releases (single-file!).

.github/workflows/publish-to-release.yml

@@ -42,7 +42,13 @@ jobs:
       run: dotnet clean ./implement/test-elm-time/test-elm-time.csproj && dotnet nuget locals all --clear
 
     - name: dotnet publish
-      run: dotnet publish -c Debug -r ${{ matrix.publish-runtime-id }} --self-contained true -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true -p:IncludeAllContentForSelfExtract=true -p:PublishReadyToRun=true -p:PublishReadyToRunShowWarnings=true  --output ./publish  ./implement/elm-time
+      run: dotnet publish -c Debug -r ${{ matrix.publish-runtime-id }} --self-contained true -p:PublishSingleFile=true -p:PublishReadyToRun=true -p:PublishReadyToRunShowWarnings=true  --output ./dotnet-build  ./implement/elm-time
+
+    - name: Copy artifacts to publish
+      shell: pwsh
+      run: |
+        New-Item -ItemType Directory -Force -Path "./publish";
+        Get-ChildItem -Path "./dotnet-build/" -Filter "elm-time*" | ForEach-Object { Copy-Item -Path $_.FullName -Destination "./publish/" }
 
     - name: Publish artifacts
       uses: actions/upload-artifact@v3

.github/workflows/test-and-publish.yml

@@ -56,7 +56,13 @@ jobs:
           ./implement/test-elm-time/TestResults/**/*.json
 
     - name: dotnet publish
-      run: dotnet publish -c Debug -r ${{ matrix.publish-runtime-id }} --self-contained true -p:PublishSingleFile=true -p:IncludeNativeLibrariesForSelfExtract=true -p:IncludeAllContentForSelfExtract=true -p:PublishReadyToRun=true -p:PublishReadyToRunShowWarnings=true  --output ./publish  ./implement/elm-time
+      run: dotnet publish -c Debug -r ${{ matrix.publish-runtime-id }} --self-contained true -p:PublishSingleFile=true  -p:PublishReadyToRun=true -p:PublishReadyToRunShowWarnings=true  --output ./dotnet-build  ./implement/elm-time
+
+    - name: Copy artifacts to publish
+      shell: pwsh
+      run: |
+        New-Item -ItemType Directory -Force -Path "./publish";
+        Get-ChildItem -Path "./dotnet-build/" -Filter "elm-time*" | ForEach-Object { Copy-Item -Path $_.FullName -Destination "./publish/" }
 
     - name: Publish artifacts
       uses: actions/upload-artifact@v3
@@ -73,6 +79,9 @@ jobs:
         name: elm-time-separate-assemblies-${{ github.sha }}-${{ matrix.publish-runtime-id }}
         path: ./publish-separate-assemblies
 
+    - name: self-test
+      run: ./publish/elm-time  self-test
+
     - name: Elm App Compiler - Run tests with elm-time elm-test-rs
       # Adapt to elm-test-rs crashing on macOS
       # elm-test-rs also wrote the following before crashing:

implement/elm-time/ElmTime/JsEngine.cs

@@ -1,4 +1,4 @@
-using ElmTime.JavaScript;
+using ElmTime.JavaScript;
 using JavaScriptEngineSwitcher.V8;
 using System;
 
@@ -44,12 +44,12 @@ public static IJsEngine ConstructJsEngine() =>
 
     public static JavaScriptEngineSwitcher.Core.IJsEngine ConstructClearScriptJavaScriptEngine()
     {
-        ClearScript.EnsureNativeLibrariesAvailableForCurrentPlatform();
+        ClearScriptV8.SetupTask.Value.Wait();
 
         return new V8JsEngine(
             new V8Settings
             {
-                MaxStackUsage = (UIntPtr)(OverrideJsEngineSettingsMaxStackSize ?? 40_000_000),
+                MaxStackUsage = (nuint)(OverrideJsEngineSettingsMaxStackSize ?? 40_000_000),
             }
         );
     }

implement/elm-time/Git/LibGit2Sharp.cs

@@ -0,0 +1,61 @@
+using ElmTime.NativeDependency;
+using Pine;
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+
+namespace ElmTime.Git;
+
+public class LibGit2Sharp
+{
+    private static readonly IReadOnlyDictionary<OSPlatform, IReadOnlyList<DependencyFile>> DependenciesFilesByOs =
+        ImmutableDictionary<OSPlatform, IReadOnlyList<DependencyFile>>.Empty
+        .Add(
+            OSPlatform.Linux,
+            ImmutableList.Create(
+                new DependencyFile(
+                    HashBase16: "7ca026cf714e14fbab252d83974c04b843affe035e041aa1eda0d0bc258426be",
+                    ExpectedFileName: "libgit2-e632535.so",
+                    RemoteSources: new[] { "https://www.nuget.org/api/v2/package/LibGit2Sharp.NativeBinaries/2.0.320" })))
+        .Add(
+            OSPlatform.Windows,
+            ImmutableList.Create(
+                new DependencyFile(
+                    HashBase16: "76b97b411e73b7487825dd0f98cba7ce7f008bef3cbe8a35bf1af8c651a0f4a0",
+                    ExpectedFileName: "git2-e632535.dll",
+                    RemoteSources: new[] { "https://www.nuget.org/api/v2/package/LibGit2Sharp.NativeBinaries/2.0.320" })))
+        .Add(
+            OSPlatform.OSX,
+            ImmutableList.Create(
+                new DependencyFile(
+                    HashBase16: "9dc84237ca835b189636697cbe1439b0894303798efd95b55a3353ca4f12b1bb",
+                    ExpectedFileName: "libgit2-e632535.dylib",
+                    RemoteSources: new[] { "https://www.nuget.org/api/v2/package/LibGit2Sharp.NativeBinaries/2.0.320" })));
+
+    public static readonly Lazy<Task> SetupTask = new(() =>
+    {
+        EnsureNativeLibrariesAvailableForCurrentPlatform();
+        return Task.CompletedTask;
+    });
+
+    public static void EnsureNativeLibrariesAvailableForCurrentPlatform()
+    {
+        var setupForCurrentOs =
+            DependenciesFilesByOs.FirstOrDefault(c => RuntimeInformation.IsOSPlatform(c.Key)).Value
+            ??
+            throw new Exception("Unknown OS: " + RuntimeInformation.OSDescription);
+
+        var cacheDirectory = Path.GetDirectoryName(DotNetAssembly.ProgramExecutableFileName.Value)!;
+
+        foreach (var dependency in setupForCurrentOs)
+        {
+            NativeDependencies.SetUpDependency(
+                cacheDirectory: cacheDirectory,
+                dependency: dependency);
+        }
+    }
+}

implement/elm-time/JavaScript/ClearScript.cs

@@ -4,15 +4,15 @@
 using System.Collections.Immutable;
 using System.IO;
 using System.Linq;
-using System.Reflection;
 using System.Runtime.InteropServices;
-using System.Security.Cryptography;
+using System.Threading.Tasks;
+using ElmTime.NativeDependency;
 
 namespace ElmTime.JavaScript;
 
-public class ClearScript
+public class ClearScriptV8
 {
-    static readonly IReadOnlyDictionary<OSPlatform, IReadOnlyList<DependencyFile>> DependenciesFilesByOs =
+    private static readonly IReadOnlyDictionary<OSPlatform, IReadOnlyList<DependencyFile>> DependenciesFilesByOs =
         ImmutableDictionary<OSPlatform, IReadOnlyList<DependencyFile>>.Empty
         .Add(
             OSPlatform.Linux,
@@ -36,104 +36,71 @@ public class ClearScript
                     ExpectedFileName: "ClearScriptV8.osx-x64.dylib",
                     RemoteSources: new[] { "https://www.nuget.org/api/v2/package/Microsoft.ClearScript.V8.Native.osx-x64/7.4.1" })));
 
-    record DependencyFile(
-        string HashBase16,
-        string ExpectedFileName,
-        IReadOnlyList<string> RemoteSources);
+    public static readonly Lazy<Task> SetupTask = new(() =>
+    {
+        EnsureNativeLibrariesAvailableForCurrentPlatform();
+        return Task.CompletedTask;
+    });
 
-    static public void EnsureNativeLibrariesAvailableForCurrentPlatform()
+    public static void EnsureNativeLibrariesAvailableForCurrentPlatform()
     {
         var setupForCurrentOs =
             DependenciesFilesByOs.FirstOrDefault(c => RuntimeInformation.IsOSPlatform(c.Key)).Value
             ??
             throw new Exception("Unknown OS: " + RuntimeInformation.OSDescription);
 
-        var clearScriptCacheDirectory =
-            Path.Combine(Filesystem.CacheDirectory, "ClearScriptV8").TrimEnd(Path.DirectorySeparatorChar) +
+        var cacheDirectory =
+            Path.Combine(Filesystem.CacheDirectory, nameof(ClearScriptV8)).TrimEnd(Path.DirectorySeparatorChar) +
             Path.DirectorySeparatorChar;
 
         if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
         {
-            clearScriptCacheDirectory =
-                Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!;
+            cacheDirectory =
+                Path.GetDirectoryName(DotNetAssembly.ProgramExecutableFileName.Value)!;
         }
 
         if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
         {
-            clearScriptCacheDirectory =
-                Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!;
+            cacheDirectory =
+                Path.GetDirectoryName(DotNetAssembly.ProgramExecutableFileName.Value)!;
         }
 
         foreach (var dependency in setupForCurrentOs)
         {
-            SetUpDependency(
-                clearScriptCacheDirectory: clearScriptCacheDirectory,
+            NativeDependencies.SetUpDependency(
+                cacheDirectory: cacheDirectory,
                 dependency: dependency);
         }
 
         AuxiliarySearchPathAsList =
-            new[] { clearScriptCacheDirectory }
+            new[] { cacheDirectory }
             .Concat(AuxiliarySearchPathAsList ?? Array.Empty<string>())
             .Distinct()
             .ToList();
     }
 
-    static void SetUpDependency(
-        string clearScriptCacheDirectory,
-        DependencyFile dependency)
-    {
-        var hash = CommonConversion.ByteArrayFromStringBase16(dependency.HashBase16);
-
-        var fileAbsolutePath = Path.Combine(clearScriptCacheDirectory, dependency.ExpectedFileName);
-
-        if (File.Exists(fileAbsolutePath))
-        {
-            var fileContent = File.ReadAllBytes(fileAbsolutePath);
-
-            var fileContentHash = SHA256.HashData(fileContent);
-
-            if (fileContentHash.SequenceEqual(hash))
-            {
-                return;
-            }
-        }
-
-        var file = BlobLibrary.GetBlobWithSHA256Cached(
-            sha256: hash,
-            getIfNotCached:
-            () =>
-            BlobLibrary.DownloadFromUrlAndExtractBlobWithMatchingHashFromListOfRemoteSources(
-                dependency.RemoteSources, hash))
-            ?? throw new Exception(
-                "Did not find dependency " + dependency.HashBase16 + " (" + dependency.ExpectedFileName + ") in any of the " +
-                dependency.RemoteSources.Count + " remote sources");
-
-        ExecutableFile.CreateAndWriteFileToPath(fileAbsolutePath, file, executable: true);
-    }
 
     /// <summary>
     /// As declared at https://github.com/microsoft/ClearScript/blob/d9a58a66e6a2a39d01e2adea9071f6a460381c8a/ClearScript/Util/MiscHelpers.cs#L133
     /// 
     /// See documentation on <see cref="Microsoft.ClearScript.HostSettings.AuxiliarySearchPath"/>
     /// </summary>
-    static char AuxiliarySearchPathSeparatorChar => ';';
+    private static char AuxiliarySearchPathSeparatorChar => ';';
 
-    static public IReadOnlyList<string>? AuxiliarySearchPathAsList
+    public static IReadOnlyList<string>? AuxiliarySearchPathAsList
     {
-        set
-        {
+        set =>
             Microsoft.ClearScript.HostSettings.AuxiliarySearchPath =
                 value is null
-                ?
-                null
-                :
-                string.Join(AuxiliarySearchPathSeparatorChar, value);
-        }
+                    ?
+                    null
+                    :
+                    string.Join(AuxiliarySearchPathSeparatorChar, value);
 
         get =>
             Microsoft.ClearScript.HostSettings.AuxiliarySearchPath switch
             {
-                string value => value.Split(AuxiliarySearchPathSeparatorChar),
+                { } value => value.Split(AuxiliarySearchPathSeparatorChar),
                 null => null
             };
     }

implement/elm-time/NativeDependency/DependencyFile.cs

@@ -0,0 +1,9 @@
+using System.Collections.Generic;
+
+namespace ElmTime.NativeDependency;
+
+
+public record DependencyFile(
+    string HashBase16,
+    string ExpectedFileName,
+    IReadOnlyList<string> RemoteSources);

implement/elm-time/NativeDependency/NativeDependencies.cs

@@ -0,0 +1,43 @@
+using Pine;
+using System;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+
+namespace ElmTime.NativeDependency;
+
+public class NativeDependencies
+{
+    public static void SetUpDependency(
+        string cacheDirectory,
+        DependencyFile dependency)
+    {
+        var hash = CommonConversion.ByteArrayFromStringBase16(dependency.HashBase16);
+
+        var fileAbsolutePath = Path.Combine(cacheDirectory, dependency.ExpectedFileName);
+
+        if (File.Exists(fileAbsolutePath))
+        {
+            var fileContent = File.ReadAllBytes(fileAbsolutePath);
+
+            var fileContentHash = SHA256.HashData(fileContent);
+
+            if (fileContentHash.SequenceEqual(hash))
+            {
+                return;
+            }
+        }
+
+        var file = BlobLibrary.GetBlobWithSHA256Cached(
+            sha256: hash,
+            getIfNotCached:
+            () =>
+            BlobLibrary.DownloadFromUrlAndExtractBlobWithMatchingHashFromListOfRemoteSources(
+                dependency.RemoteSources, hash))
+            ?? throw new Exception(
+                "Did not find dependency " + dependency.HashBase16 + " (" + dependency.ExpectedFileName + ") in any of the " +
+                dependency.RemoteSources.Count + " remote sources");
+
+        ExecutableFile.CreateAndWriteFileToPath(fileAbsolutePath, file, executable: true);
+    }
+}

implement/elm-time/Pine/BlobLibrary.cs

@@ -173,7 +173,7 @@ public static async Task<Result<string, ReadOnlyMemory<byte>>> DownloadBlobViaHt
         }
     }
 
-    static int DownloadViaHttpRetryCountDefault =>
+    private static int DownloadViaHttpRetryCountDefault =>
         /*
          * Adapt to problem observed 2023-05-21 on environment windows-2022:
          * On that environment, the HTTP request sometimes failed with a runtime exception like the following:

implement/elm-time/Pine/DotNetAssembly.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Diagnostics;
 using System.Linq;
 using System.Reflection;
 
@@ -119,4 +120,20 @@ static TreeNodeWithStringPath fromFile(Microsoft.Extensions.FileProviders.IFileI
 
         return memoryStream.ToArray();
     }
+
+    public static readonly Lazy<string> ProgramExecutableFileName = new(() =>
+    // Assembly.GetExecutingAssembly().Location for cases where process comes from `dotnet test`
+    Assembly.GetExecutingAssembly().Location switch
+    {
+        { Length: > 0 } executingAssemblyLocation =>
+        executingAssemblyLocation,
+
+        _ =>
+        Environment.ProcessPath ??
+        /*
+         * Do not rely on Environment.ProcessPath because it is often null on Linux:
+         * https://github.com/dotnet/runtime/issues/66323
+         * */
+        Process.GetCurrentProcess().MainModule!.FileName
+    });
 }

implement/elm-time/Pine/LoadFromGitHubOrGitLab.cs

@@ -111,6 +111,8 @@ public static Result<string, LoadFromUrlSuccess> LoadFromUrl(
         string sourceUrl,
         Func<GetRepositoryFilesPartialForCommitRequest, Result<string, IImmutableDictionary<IReadOnlyList<string>, ReadOnlyMemory<byte>>>> getRepositoryFilesPartialForCommit)
     {
+        ElmTime.Git.LibGit2Sharp.SetupTask.Value.Wait();
+
         var parsedUrl = ParseUrl(sourceUrl);
 
         if (parsedUrl == null)
@@ -362,6 +364,8 @@ public static Result<string, IImmutableDictionary<IReadOnlyList<string>, ReadOnl
         string cloneUrl,
         string? branchName)
     {
+        ElmTime.Git.LibGit2Sharp.SetupTask.Value.Wait();
+
         var refName = RefCanonicalNameFromPathComponentInGitHubRepository(branchName);
 
         return
@@ -418,7 +422,7 @@ public static Result<string, IImmutableDictionary<IReadOnlyList<string>, ReadOnl
         }
     }
 
-    static string ResolveLinkTargetFullNameIncludingParents(string directory)
+    private static string ResolveLinkTargetFullNameIncludingParents(string directory)
     {
         if (directory == Path.GetPathRoot(directory))
             return directory;