Precompute all possible slider states

Project.toml

@@ -9,6 +9,7 @@ Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 BetterFileWatching = "c9fd44ac-77b5-486c-9482-9798bd063cc6"
 Configurations = "5218b696-f38b-4ac9-8b61-a12ec717816d"
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 FromFile = "ff7dd447-1dcb-4ce3-b8ac-22a812192de7"
 Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2"
 Glob = "c27321d9-0574-5035-807b-f59d2c89b15c"
@@ -17,12 +18,16 @@ HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
+Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Pluto = "c3e4b0f8-55cb-11ea-2926-15256bba5781"
 PlutoDependencyExplorer = "72656b73-756c-7461-726b-72656b6b696b"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 TerminalLoggers = "5d786b92-1e48-4d6f-9151-6b4477ca9bed"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
@@ -34,16 +39,19 @@ BetterFileWatching = "^0.1.2"
 Configurations = "0.16, 0.17"
 Distributed = "1"
 FromFile = "0.1"
+Distributions = "0.25"
 Git = "1"
 Glob = "1"
 GracefulPkg = "2"
 HTTP = "^1.0.2"
+OrderedCollections = "1"
 JSON = "0.21"
 LibGit2 = "1"
 Logging = "1"
 Pkg = "1"
 Pluto = "0.20.6"
 PlutoDependencyExplorer = "1"
+Random = "1"
 SHA = "0.7, 1"
 Serialization = "1"
 Sockets = "1"
@@ -53,8 +61,8 @@ UUIDs = "1"
 julia = "1.10"
 
 [extras]
-Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Deno_jll = "04572ae6-984a-583e-9378-9577a1c2574d"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test", "Random"]
+test = ["Deno_jll", "Test"]

src/Actions.jl

@@ -1,12 +1,15 @@
-import Pluto: Pluto, without_pluto_file_extension, generate_html, @asynclog
+import Pluto:
+    Pluto, without_pluto_file_extension, generate_html, @asynclog, withtoken, Firebasey
 using Base64
 using FromFile
+import HTTP.URIs
 
 @from "./MoreAnalysis.jl" import bound_variable_connections_graph
 @from "./Export.jl" import try_get_exact_pluto_version,
     try_fromcache, try_tocache, write_statefile
 @from "./Types.jl" import NotebookSession, RunningNotebook, FinishedNotebook, RunResult
 @from "./Configuration.jl" import PlutoDeploySettings, is_glob_match
+@from "./precomputed/index.jl" import generate_precomputed_staterequests
 @from "./PlutoHash.jl" import plutohash
 @from "./PathUtils.jl" import to_local_path, to_url_path
 
@@ -67,8 +70,7 @@ function process(
             url_path new_hash s.desired_hash
     end
 
-    keep_running =
-        settings.SliderServer.enabled &&
+    keep_running = (settings.SliderServer.enabled || settings.Precompute.enabled) &&
         !is_glob_match(url_path, settings.SliderServer.exclude) &&
         occursin("@bind", jl_contents)
     skip_cache = keep_running || is_glob_match(url_path, settings.Export.ignore_cache)
@@ -121,9 +123,20 @@ function process(
         )
     end
 
+    new_session = NotebookSession(; path=url_path, current_hash=new_hash, desired_hash=s.desired_hash, run)
+    if settings.Precompute.enabled
+        generate_precomputed_staterequests(
+            new_session;
+            settings,
+            pluto_session=server_session,
+            output_dir,
+        )
+        # TODO shutdown
+    end
+
     @info "### ✓ $(progress) Ready" path = url_path new_hash t_elapsed
 
-    NotebookSession(; path=url_path, current_hash=new_hash, desired_hash=s.desired_hash, run)
+    new_session
 end
 
 ###
@@ -213,7 +226,11 @@ function generate_static_export(
 
     slider_server_running_somewhere =
         settings.Export.slider_server_url !== nothing ||
-        (settings.SliderServer.serve_static_export_folder && settings.SliderServer.enabled)
+        (
+            settings.SliderServer.serve_static_export_folder &&
+            settings.SliderServer.enabled
+        ) ||
+        settings.Precompute.enabled
 
     notebookfile_js = if settings.Export.offer_binder || slider_server_running_somewhere
         if settings.Export.baked_notebookfile

src/Configuration.jl

@@ -1,6 +1,7 @@
 using Configurations
 import Pluto
-export SliderServerSettings, ExportSettings, PlutoDeploySettings, get_configuration
+export SliderServerSettings,
+    ExportSettings, PrecomputeSettings, PlutoDeploySettings, get_configuration
 using FromFile
 import Glob
 @from "./ConfigurationDocs.jl" import @extract_docs, get_kwdocs, list_options_md
@@ -24,6 +25,17 @@ import Glob
     cache_control::String = "public, max-age=315600000, immutable"
 end
 
+
+@extract_docs @option struct PrecomputeSettings
+    "Precompute slider server requests?"
+    enabled::Bool = false
+    "List of notebook files to skip precomputation. Provide paths relative to `start_dir`."
+    exclude::Vector{String} = String[]
+    "Whether or not to partially precompute notebooks. If `true`, notebooks will only be precomputed if **all** their sliders can be precomputed"
+    only_fully::Bool = false
+    max_filesize_per_group::Integer = 1_000_000
+end
+
 @extract_docs @option struct ExportSettings
     "Generate static HTML files? This setting can only be `false` if you are also running a slider server."
     enabled::Bool = true
@@ -59,6 +71,7 @@ end
 @option struct PlutoDeploySettings
     SliderServer::SliderServerSettings = SliderServerSettings()
     Export::ExportSettings = ExportSettings()
+    Precompute::PrecomputeSettings = PrecomputeSettings()
     Pluto::Pluto.Configuration.Options = Pluto.Configuration.Options()
 end
 

src/HTTPRouter.jl

@@ -3,18 +3,13 @@ using FromFile
 import Pluto
 import Pluto:
     ServerSession,
-    Firebasey,
-    Token,
-    withtoken,
-    pluto_file_extensions,
-    without_pluto_file_extension
+    Firebasey
 using HTTP
 using Sockets
-import JSON
-import PlutoDependencyExplorer
 
 
 
+@from "./run_bonds.jl" import run_bonds_get_patches, patches_to_response_data
 @from "./IndexJSON.jl" import generate_index_json
 @from "./IndexHTML.jl" import generate_temp_index_html
 @from "./Types.jl" import NotebookSession, RunningNotebook, FinishedNotebook
@@ -41,7 +36,7 @@ function make_router(
 
     with_cacheable_configured! = with_cacheable!(settings.SliderServer.cache_control)
 
-    function get_sesh(request::HTTP.Request)
+    function get_sesh(request::HTTP.Request)::Union{Nothing,NotebookSession}
         uri = HTTP.URI(request.target)
 
         parts = HTTP.URIs.splitpath(uri.path)
@@ -122,130 +117,23 @@ function make_router(
             end
             t3 = time()
 
-            names::Vector{Symbol} = Symbol.(keys(bonds))
-
-            # "explicits" defaults to "all bonds"
-            explicits = @something(explicits, Set(names))
-
-            names_original = names
-            names =
-                begin
-                    # REMOVE ALL NAMES that depend on 
-                    # an explicit bond
-                    # 
-                    # Because its new value might no longer be valid.
-                    # 
-                    # For exaxmple:
-                    # @bind xx Slider(1:100)
-                    # @bind yy Slider(xx:100)
-                    # 
-                    # (ignore bond transformatinos for now)
-                    # 
-                    # The sliders will be set on (1,1) initially.
-                    # The user moves the first slider, giving (10,1).
-                    # The value for `y` will be sent, but this should be ignored. Because it was generated from an outdated bond.
-
-                    first_layer = PlutoDependencyExplorer.where_referenced(
-                        notebook.topology,
-                        explicits,
-                    )
-
-                    next_layers = Pluto.MoreAnalysis.downstream_recursive(
-                        notebook.topology,
-                        first_layer,
-                    )
-
-                    # all cells that depend on an explicit bond
-                    cells_depending_on_explicits = union!(first_layer, next_layers)
-
-                    # remove any variable `n` from `names` if...
-                    filter(names) do n
-                        !(
-                            # ...`n` depends on an explicit bond.
-                            any(cells_depending_on_explicits) do c
-                                n in notebook.topology.nodes[c].definitions
-                            end
-                        )
-                    end
-                end
-
-            t35 = time()
-
-
-            id(c) = c.cell_id
-
-            @debug "Analysis" names names_original id.(cells_depending_on_explicits)
-
-            new_state = withtoken(sesh.run.token) do
-                try
-                    # Set the bond values. We don't need to merge dicts here because the old bond values will never be used.
-                    notebook.bonds = bonds
-
-                    # Run the bonds!
-                    topological_order = Pluto.set_bond_values_reactive(
-                        session=server_session,
-                        notebook=notebook,
-                        bound_sym_names=names,
-                        is_first_values=[false for _n in names], # because requests should be stateless. We might want to do something special for the (actual) initial request (containing every initial bond value) in the future.
-                        run_async=false,
-                    )::Pluto.TopologicalOrder
-
-                    @debug "Finished running!" length(topological_order.runnable)
-
-                    Pluto.notebook_to_js(notebook)
-                catch e
-                    @error "Failed to set bond values" exception = (e, catch_backtrace())
-                    nothing
-                end
-            end
-            new_state === nothing && return (
+            result = run_bonds_get_patches(
+                server_session,
+                sesh.run,
+                bonds,
+                explicits,
+            )
+            result === nothing && return (
                 HTTP.Response(500, "Failed to set bond values") |>
                 with_cors! |>
                 with_not_cacheable!
             )
 
-            t4 = time()
-
-            # We only want to send state updates about...
-            function only_relevant(state)
-                new = copy(state)
-                # ... the cells that just ran and ...
-                new["cell_results"] = filter(state["cell_results"]) do (id, cell_state)
-                    id ∈ (c.cell_id for c in cells_depending_on_explicits)
-                end
-                # ... nothing about bond values, because we don't want to synchronize among clients. and...
-                delete!(new, "bonds")
-                # ... we ignore changes to the status tree caused by a running bonds.
-                delete!(new, "status_tree")
-                new
-            end
-
-            patches = let
-                notebook_patches = Firebasey.diff(
-                    only_relevant(sesh.run.original_state),
-                    only_relevant(new_state),
-                )
-
-                # Remove bonds that depend on explicit bonds. Because this means that the bond was re-created during this run, and its value must be reset,
-                bond_patches = [
-                    Firebasey.RemovePatch(["bonds", string(k)]) for
-                    k in keys(bonds) if k ∉ names
-                ]
-
-                @debug "patches" notebook_patches bond_patches
-
-                union!(notebook_patches, bond_patches)
-            end
-
-            patches_as_dicts::Array{Dict} = Firebasey._convert(Array{Dict}, patches)
+            (; patches, t35, t4 ) = result
 
             t5 = time()
-
-            response_data = Pluto.pack(
-                Dict{String,Any}(
-                    "patches" => patches_as_dicts,
-                ),
-            )
+            response_data = patches_to_response_data(patches)
+
             @debug "Sending patches" Pluto.unpack(response_data)
 
             t6 = time()

src/PlutoSliderServer.jl

@@ -20,6 +20,7 @@ export cache_filename
 @from "./ReloadFolder.jl" import update_sessions!, select
 @from "./HTTPRouter.jl" import make_router, ReferrerMiddleware
 @from "./gitpull.jl" import fetch_pull
+@from "./precomputed/debug.jl" import start_debugging
 
 @from "./PlutoHash.jl" import plutohash, base64urlencode, base64urldecode
 export plutohash, base64urlencode, base64urldecode
@@ -553,4 +554,5 @@ function kind_of_debounced(f)
 end
 
 
+
 end