From 67658db8f7e2ed50a9dd2a3ffcfaba2e20c7615d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michael=20R=C3=A4tzel?= Date: Fri, 19 Apr 2024 14:43:40 +0000 Subject: [PATCH] Make migrations easier with database store replication default Expand the Docker image to default to replicate from the previously used default storage location in cases where the new store container is empty. --- implement/Dockerfile | 9 ++- implement/pine/Pine/FileStore.cs | 37 --------- implement/pine/Pine/FileStoreExtension.cs | 98 +++++++++++++++++++++++ implement/pine/Program.cs | 16 +++- implement/pine/RunServer.cs | 21 ++++- implement/pine/Test/SelfTest.cs | 1 + implement/pine/pine.csproj | 4 +- 7 files changed, 140 insertions(+), 46 deletions(-) create mode 100644 implement/pine/Pine/FileStoreExtension.cs diff --git a/implement/Dockerfile b/implement/Dockerfile index bcce6271..2d56b435 100644 --- a/implement/Dockerfile +++ b/implement/Dockerfile @@ -22,10 +22,15 @@ RUN apt install -y git COPY ./example-apps/docker-image-default-app /docker-image-default-app/ -RUN dotnet "/pine/dotnet/pine.dll" deploy /docker-image-default-app/ /pine/process-store --init-app-state +RUN dotnet "/pine/dotnet/pine.dll" deploy /docker-image-default-app/ /pine-vm/process-store --init-app-state WORKDIR /pine -ENTRYPOINT ["dotnet", "/pine/dotnet/pine.dll", "run-server", "--process-store=/pine/process-store"] +# Mounting a docker volume can shadow the previous state of the process store: +# +# docker volume create docker-volume-name +# docker run --mount 'source=docker-volume-name,destination=/pine-vm/process-store' .... + +ENTRYPOINT ["dotnet", "/pine/dotnet/pine.dll", "run-server", "--process-store=/pine-vm/process-store", "--process-store-readonly=/elm-time/process-store"] # ENV APPSETTING_adminPassword="password-for-admin-interface" diff --git a/implement/pine/Pine/FileStore.cs b/implement/pine/Pine/FileStore.cs index cc42755a..60d712b2 100644 --- a/implement/pine/Pine/FileStore.cs +++ b/implement/pine/Pine/FileStore.cs @@ -289,40 +289,3 @@ public class EmptyFileStoreReader : IFileStoreReader public IEnumerable> ListFilesInDirectory(IImmutableList directoryPath) => []; } - -public static class FileStoreExtension -{ - public static IEnumerable> ListFiles(this IFileStoreReader fileStore) => - fileStore.ListFilesInDirectory(ImmutableList.Empty); - - public static IFileStoreReader WithMappedPath( - this IFileStoreReader originalFileStore, Func, IImmutableList> pathMap) => - new DelegatingFileStoreReader - ( - GetFileContentDelegate: originalPath => originalFileStore.GetFileContent(pathMap(originalPath)), - ListFilesInDirectoryDelegate: originalPath => originalFileStore.ListFilesInDirectory(pathMap(originalPath)) - ); - - public static IFileStoreReader ForSubdirectory(this IFileStoreReader originalFileStore, string directoryName) => - ForSubdirectory(originalFileStore, ImmutableList.Create(directoryName)); - - public static IFileStoreReader ForSubdirectory( - this IFileStoreReader originalFileStore, IEnumerable directoryPath) => - WithMappedPath(originalFileStore, originalPath => originalPath.InsertRange(0, directoryPath)); - - public static IFileStoreWriter WithMappedPath( - this IFileStoreWriter originalFileStore, Func, IImmutableList> pathMap) => - new DelegatingFileStoreWriter - ( - SetFileContentDelegate: pathAndFileContent => originalFileStore.SetFileContent(pathMap(pathAndFileContent.path), pathAndFileContent.fileContent), - AppendFileContentDelegate: pathAndFileContent => originalFileStore.AppendFileContent(pathMap(pathAndFileContent.path), pathAndFileContent.fileContent), - DeleteFileDelegate: originalPath => originalFileStore.DeleteFile(pathMap(originalPath)) - ); - - public static IFileStoreWriter ForSubdirectory(this IFileStoreWriter originalFileStore, string directoryName) => - ForSubdirectory(originalFileStore, ImmutableList.Create(directoryName)); - - public static IFileStoreWriter ForSubdirectory( - this IFileStoreWriter originalFileStore, IEnumerable directoryPath) => - WithMappedPath(originalFileStore, originalPath => originalPath.InsertRange(0, directoryPath)); -} diff --git a/implement/pine/Pine/FileStoreExtension.cs b/implement/pine/Pine/FileStoreExtension.cs new file mode 100644 index 00000000..61c36c9d --- /dev/null +++ b/implement/pine/Pine/FileStoreExtension.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.Collections.Immutable; + +namespace Pine; + +public static class FileStoreExtension +{ + public static IEnumerable> ListFiles(this IFileStoreReader fileStore) => + fileStore.ListFilesInDirectory([]); + + public static IFileStoreReader WithMappedPath( + this IFileStoreReader originalFileStore, Func, IImmutableList> pathMap) => + new DelegatingFileStoreReader + ( + GetFileContentDelegate: originalPath => originalFileStore.GetFileContent(pathMap(originalPath)), + ListFilesInDirectoryDelegate: originalPath => originalFileStore.ListFilesInDirectory(pathMap(originalPath)) + ); + + public static IFileStoreReader ForSubdirectory(this IFileStoreReader originalFileStore, string directoryName) => + ForSubdirectory(originalFileStore, ImmutableList.Create(directoryName)); + + public static IFileStoreReader ForSubdirectory( + this IFileStoreReader originalFileStore, IEnumerable directoryPath) => + WithMappedPath(originalFileStore, originalPath => originalPath.InsertRange(0, directoryPath)); + + public static IFileStoreWriter WithMappedPath( + this IFileStoreWriter originalFileStore, Func, IImmutableList> pathMap) => + new DelegatingFileStoreWriter + ( + SetFileContentDelegate: + pathAndFileContent => + originalFileStore.SetFileContent(pathMap(pathAndFileContent.path), pathAndFileContent.fileContent), + + AppendFileContentDelegate: + pathAndFileContent => + originalFileStore.AppendFileContent(pathMap(pathAndFileContent.path), pathAndFileContent.fileContent), + + DeleteFileDelegate: + originalPath => originalFileStore.DeleteFile(pathMap(originalPath)) + ); + + public static IFileStoreWriter ForSubdirectory(this IFileStoreWriter originalFileStore, string directoryName) => + ForSubdirectory(originalFileStore, ImmutableList.Create(directoryName)); + + public static IFileStoreWriter ForSubdirectory( + this IFileStoreWriter originalFileStore, IEnumerable directoryPath) => + WithMappedPath(originalFileStore, originalPath => originalPath.InsertRange(0, directoryPath)); + + public static IFileStore MergeReader( + this IFileStore primary, + IFileStoreReader secondary, + bool promoteOnReadFileContentFromSecondary) + { + ReadOnlyMemory? GetFileContent(IImmutableList path) + { + var primaryContent = primary.GetFileContent(path); + + if (primaryContent is not null) + { + return primaryContent; + } + + var secondaryContent = secondary.GetFileContent(path); + + if (secondaryContent.HasValue && promoteOnReadFileContentFromSecondary) + { + primary.SetFileContent(path, secondaryContent.Value); + } + + return secondaryContent; + } + + var mergedReader = + new DelegatingFileStoreReader + ( + GetFileContentDelegate: + GetFileContent, + + ListFilesInDirectoryDelegate: + path => [.. secondary.ListFilesInDirectory(path), .. primary.ListFilesInDirectory(path)] + ); + + return new FileStoreFromWriterAndReader(primary, mergedReader); + } + + public static IFileStoreReader MergeReader( + this IFileStoreReader primary, + IFileStoreReader secondary) => + new DelegatingFileStoreReader + ( + GetFileContentDelegate: + path => primary.GetFileContent(path) ?? secondary.GetFileContent(path), + + ListFilesInDirectoryDelegate: + path => [.. secondary.ListFilesInDirectory(path), .. primary.ListFilesInDirectory(path)] + ); +} diff --git a/implement/pine/Program.cs b/implement/pine/Program.cs index 89cba4c8..7e1c04b0 100644 --- a/implement/pine/Program.cs +++ b/implement/pine/Program.cs @@ -18,7 +18,7 @@ namespace ElmTime; public class Program { - public static string AppVersionId => "0.3.0"; + public static string AppVersionId => "0.3.1"; private static int AdminInterfaceDefaultPort => 4000; @@ -280,7 +280,18 @@ private static CommandLineApplication AddRunServerCommand( var adminUrlsDefault = "http://*:" + AdminInterfaceDefaultPort; - var processStoreOption = runServerCommand.Option("--process-store", "Directory in the file system to contain the process store.", CommandOptionType.SingleValue); + var processStoreOption = + runServerCommand.Option( + "--process-store", + "Directory in the file system to contain the process store.", + CommandOptionType.SingleValue); + + var processStoreReadonlyOption = + runServerCommand.Option( + "--process-store-readonly", + "If the primary process store is empty at startup, the system will try to replicate from this location.", + CommandOptionType.SingleValue); + var deletePreviousProcessOption = runServerCommand.Option("--delete-previous-process", "Delete the previous backend process found in the given store. If you don't use this option, the server restores the process from the persistent store on startup.", CommandOptionType.NoValue); var adminUrlsOption = runServerCommand.Option("--admin-urls", "URLs for the admin interface. The default is " + adminUrlsDefault + ".", CommandOptionType.SingleValue); var adminPasswordOption = runServerCommand.Option("--admin-password", "Password for the admin interface at '--admin-urls'.", CommandOptionType.SingleValue); @@ -307,6 +318,7 @@ private static CommandLineApplication AddRunServerCommand( var webHost = RunServer.BuildWebHostToRunServer( processStorePath: processStorePath, + processStoreReadonlyPath: processStoreReadonlyOption.Value(), adminInterfaceUrls: adminInterfaceUrls, adminPassword: adminPasswordOption.Value(), publicAppUrls: publicAppUrls, diff --git a/implement/pine/RunServer.cs b/implement/pine/RunServer.cs index 8d613daf..bbb3ed5d 100644 --- a/implement/pine/RunServer.cs +++ b/implement/pine/RunServer.cs @@ -16,6 +16,7 @@ public class RunServer { public static IWebHost BuildWebHostToRunServer( string? processStorePath, + string? processStoreReadonlyPath, string? adminInterfaceUrls, string? adminPassword, IReadOnlyList? publicAppUrls, @@ -76,6 +77,16 @@ IFileStore buildProcessStoreFileStore() var processStoreFileStore = buildProcessStoreFileStore(); + if (processStoreReadonlyPath is not null) + { + Console.WriteLine("Merging read-only process store from '" + processStoreReadonlyPath + "'."); + + processStoreFileStore = + processStoreFileStore?.MergeReader( + new FileStoreFromSystemIOFile(processStoreReadonlyPath), + promoteOnReadFileContentFromSecondary: true); + } + if (copyProcess is not null) { var copyFiles = @@ -90,10 +101,14 @@ IFileStore buildProcessStoreFileStore() var javaScriptEngineFactory = elmEngineType switch { - ElmInteractive.ElmEngineType.JavaScript_Jint => JavaScriptEngineJintOptimizedForElmApps.Create, - ElmInteractive.ElmEngineType.JavaScript_V8 => new Func(JavaScriptEngineFromJavaScriptEngineSwitcher.ConstructJavaScriptEngine), + ElmInteractive.ElmEngineType.JavaScript_Jint => + JavaScriptEngineJintOptimizedForElmApps.Create, + + ElmInteractive.ElmEngineType.JavaScript_V8 => + new Func(JavaScriptEngineFromJavaScriptEngineSwitcher.ConstructJavaScriptEngine), - object other => throw new NotImplementedException("Engine type not implemented here: " + other) + object other => + throw new NotImplementedException("Engine type not implemented here: " + other) }; if (deployApp is not null) diff --git a/implement/pine/Test/SelfTest.cs b/implement/pine/Test/SelfTest.cs index 2fda6b86..8124f742 100644 --- a/implement/pine/Test/SelfTest.cs +++ b/implement/pine/Test/SelfTest.cs @@ -72,6 +72,7 @@ public static int RunWebServerTest() using var webHost = RunServer.BuildWebHostToRunServer( processStorePath: null, + processStoreReadonlyPath: null, adminInterfaceUrls: null, adminPassword: null, publicAppUrls: ["http://localhost:" + serverHttpPort], diff --git a/implement/pine/pine.csproj b/implement/pine/pine.csproj index 5a0cd8ea..9405e0d3 100644 --- a/implement/pine/pine.csproj +++ b/implement/pine/pine.csproj @@ -4,8 +4,8 @@ Exe net8.0 pine - 0.3.0 - 0.3.0 + 0.3.1 + 0.3.1 enable true