ADR 0045 D2: substrate v2b wiring — shared base shm + handler CONTINUE arm

.github/workflows/ci.yml

@@ -626,6 +626,7 @@ jobs:
             --test file_restore_shared_rss \
             --test stock_fork_snapshot_compat \
             --test substrate_kernel_capabilities \
+            --test substrate_uffd_base \
             --run-ignored ignored-only \
             --no-fail-fast \
             --test-threads=1

Tiltfile

@@ -397,6 +397,12 @@ def host_agent_resource(name, grpc_port, metrics_port, work_dir, nbd_csv):
         # where the handler is present. (Base-create stays code-default
         # `file`, which needs no handler.)
         'ENGRAM_FC_RESTORE_MODE': env_or('ENGRAM_FC_RESTORE_MODE', 'file'),
+        # ADR 0045 substrate (v2b): point at a tmpfs dir (e.g.
+        # /dev/shm/engram) to back Uffd restores with a shared
+        # per-template base shm. Empty = off (stock anonymous Uffd).
+        # Must be listed here: the host-agent runs under sudo
+        # --preserve-env=<these keys>, which scrubs unlisted vars.
+        'ENGRAM_FC_UFFD_BASE_DIR': env_or('ENGRAM_FC_UFFD_BASE_DIR', ''),
         # gRPC plumbing — coord dials advertise, host-agent listens on
         # bind. Same machine in dev, so loopback works for both.
         'ENGRAM_GRPC_LISTEN_ADDR': '127.0.0.1:' + grpc_port,

crates/engram-host-agent/src/live_attach.rs

@@ -271,6 +271,7 @@ mod tests {
             guest_otel_endpoint: None,
             firecracker_bin: PathBuf::from("/nonexistent/firecracker"),
             uffd_handler_bin: PathBuf::from("/nonexistent/engram-uffd-handler"),
+            uffd_base_dir: None,
             restore_mode: engram_sandbox_firecracker::RestoreMode::File,
             base_restore_mode: None,
             track_dirty_pages: false,

crates/engram-host-agent/src/main.rs

@@ -331,6 +331,13 @@ async fn main() -> Result<(), HostAgentError> {
             if let Ok(p) = std::env::var("ENGRAM_FC_UFFD_HANDLER_BIN") {
                 fc_cfg.uffd_handler_bin = p.into();
             }
+            // ADR 0045 unified memory substrate (v2b): when
+            // ENGRAM_FC_UFFD_BASE_DIR points at a tmpfs dir, Uffd-mode
+            // restores back guest memory MAP_PRIVATE on a per-template
+            // base shm file there — canonical pages become one shared
+            // page-cache copy per host (the D2 rollout gate; D3/D4
+            // parity flips retire the knob).
+            fc_cfg.uffd_base_dir = engram_sandbox_firecracker::uffd_base_dir_from_env();
             // ADR 0014 M1.12: each FC host maintains a 16 MiB empty
             // ext4 stub harness that warm-pool restore points the
             // harness symlink at. Content-identical to the one the

crates/engram-sandbox-firecracker/src/client.rs

@@ -283,6 +283,8 @@ impl FirecrackerClient {
             },
             enable_diff_snapshots,
             resume_vm,
+            // Substrate base backing is a Uffd-mode concept (ADR 0045 v2b).
+            uffd_base_file: None,
         };
         self.put("/snapshot/load", &body).await
     }
@@ -300,8 +302,14 @@ impl FirecrackerClient {
         state_path: &Path,
         uffd_uds_path: &Path,
     ) -> Result<(), SandboxError> {
-        self.load_snapshot_uffd_inner(state_path, uffd_uds_path, /*resume_vm=*/ true, false)
-            .await
+        self.load_snapshot_uffd_inner(
+            state_path,
+            uffd_uds_path,
+            /*resume_vm=*/ true,
+            false,
+            None,
+        )
+        .await
     }
 
     /// UFFD-backed load that leaves the VM paused. Caller must
@@ -316,8 +324,14 @@ impl FirecrackerClient {
         state_path: &Path,
         uffd_uds_path: &Path,
     ) -> Result<(), SandboxError> {
-        self.load_snapshot_uffd_inner(state_path, uffd_uds_path, /*resume_vm=*/ false, false)
-            .await
+        self.load_snapshot_uffd_inner(
+            state_path,
+            uffd_uds_path,
+            /*resume_vm=*/ false,
+            false,
+            None,
+        )
+        .await
     }
 
     /// UFFD-backed load with explicit `resume_vm` /
@@ -329,9 +343,16 @@ impl FirecrackerClient {
         uffd_uds_path: &Path,
         resume_vm: bool,
         enable_diff_snapshots: bool,
+        uffd_base_file: Option<&Path>,
     ) -> Result<(), SandboxError> {
-        self.load_snapshot_uffd_inner(state_path, uffd_uds_path, resume_vm, enable_diff_snapshots)
-            .await
+        self.load_snapshot_uffd_inner(
+            state_path,
+            uffd_uds_path,
+            resume_vm,
+            enable_diff_snapshots,
+            uffd_base_file,
+        )
+        .await
     }
 
     async fn load_snapshot_uffd_inner(
@@ -340,6 +361,7 @@ impl FirecrackerClient {
         uffd_uds_path: &Path,
         resume_vm: bool,
         enable_diff_snapshots: bool,
+        uffd_base_file: Option<&Path>,
     ) -> Result<(), SandboxError> {
         let body = SnapshotLoadBody {
             snapshot_path: state_path.to_string_lossy().into_owned(),
@@ -349,6 +371,7 @@ impl FirecrackerClient {
             },
             enable_diff_snapshots,
             resume_vm,
+            uffd_base_file: uffd_base_file.map(|p| p.to_string_lossy().into_owned()),
         };
         self.put("/snapshot/load", &body).await
     }
@@ -703,6 +726,15 @@ struct SnapshotLoadBody {
     /// `true` makes Firecracker resume the guest immediately after
     /// load (no separate `PATCH /vm Resumed` needed).
     resume_vm: bool,
+    /// ADR 0045 substrate (v2b): when set with the Uffd backend, the
+    /// forked FC creates guest memory as `MAP_PRIVATE` of this shmem
+    /// base file and registers UFFD `MISSING|MINOR`, letting the
+    /// handler share base-identical pages across same-template VMs via
+    /// `UFFDIO_CONTINUE`. `skip_serializing_if` keeps the body
+    /// byte-identical to stock when unset (stock FC and pre-v2 forks
+    /// `deny_unknown_fields` this struct).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    uffd_base_file: Option<String>,
 }
 
 /// How memory is supplied during snapshot load.
@@ -1120,6 +1152,7 @@ mod tests {
             },
             enable_diff_snapshots: false,
             resume_vm: true,
+            uffd_base_file: None,
         };
         let v: serde_json::Value = serde_json::to_value(&body).unwrap();
         assert_eq!(v["snapshot_path"], "/snap/state.bin");

crates/engram-sandbox-firecracker/src/lib.rs

@@ -208,6 +208,15 @@ pub struct FirecrackerConfig {
     /// when `restore_mode == Uffd`. Defaults to `engram-uffd-handler`
     /// (resolved via PATH).
     pub uffd_handler_bin: PathBuf,
+    /// ADR 0045 unified memory substrate (v2b). When set, Uffd-mode
+    /// restores back guest memory `MAP_PRIVATE` on a per-template base
+    /// shm file under this directory (`<dir>/<manifest>-v<n>.base`,
+    /// created + lazily populated by the handler), registered
+    /// `MISSING|MINOR` by the forked FC — base-identical pages are
+    /// then shared page-cache across same-template VMs via
+    /// `UFFDIO_CONTINUE`. MUST point at a tmpfs/shmem mount (MINOR
+    /// faults are shmem-only). `None` ⇒ stock anonymous Uffd restore.
+    pub uffd_base_dir: Option<PathBuf>,
     /// How to wire memory on snapshot restore. File mode synchronously
     /// reads memory.bin (slow, simple, no extra processes). Uffd mode
     /// spawns engram-uffd-handler and serves pages on demand (fast,
@@ -406,6 +415,20 @@ pub fn base_restore_mode_from_env() -> Option<RestoreMode> {
     }
 }
 
+/// ADR 0045 substrate (v2b): per-template base-shm directory for
+/// Uffd-mode restores from `ENGRAM_FC_UFFD_BASE_DIR`. Unset/empty ⇒
+/// `None` (stock anonymous Uffd restore — the D2 rollout gate; the
+/// D3/D4 parity flips make this the one path and retire the knob).
+/// The directory must live on tmpfs/shmem (e.g. `/dev/shm/engram` on
+/// the dev VM, the node-prep tmpfs in prod) — UFFD minor faults are
+/// shmem-only.
+pub fn uffd_base_dir_from_env() -> Option<PathBuf> {
+    match std::env::var("ENGRAM_FC_UFFD_BASE_DIR") {
+        Ok(s) if !s.trim().is_empty() => Some(PathBuf::from(s)),
+        _ => None,
+    }
+}
+
 /// ADR 0009 §6 errors from `FirecrackerBackend::reattach_sandbox`.
 /// Distinct from `SandboxError` so the live-attach driver can
 /// distinguish "FC truly gone, fall through to path 2" from "operator
@@ -507,6 +530,7 @@ impl FirecrackerConfig {
             guest_otel_endpoint: None,
             firecracker_bin: PathBuf::from("firecracker"),
             uffd_handler_bin: PathBuf::from("engram-uffd-handler"),
+            uffd_base_dir: None,
             restore_mode: RestoreMode::File,
             // ADR 0022: inherit `restore_mode` for base-create until prod
             // opts in via ENGRAM_FC_BASE_RESTORE_MODE.
@@ -1502,6 +1526,22 @@ impl FirecrackerBackend {
     /// Returns the live `Child` so the caller can hold it for the
     /// VM's lifetime.
     #[allow(clippy::too_many_arguments)]
+    /// ADR 0045 substrate (v2b): the per-template base shm path for a
+    /// canonical manifest, or `None` when the substrate is off. Both the
+    /// handler spawn (creates + populates it) and the FC load (maps it
+    /// `MAP_PRIVATE`) derive the path through here so they can't diverge.
+    fn uffd_base_path(
+        &self,
+        canonical_ref: &engram_core::types::manifest::ManifestRef,
+    ) -> Option<PathBuf> {
+        self.config.uffd_base_dir.as_ref().map(|dir| {
+            dir.join(format!(
+                "{}-v{}.base",
+                canonical_ref.manifest_id, canonical_ref.version
+            ))
+        })
+    }
+
     #[tracing::instrument(name = "fc.spawn_uffd_handler", skip_all)]
     async fn spawn_uffd_handler(
         &self,
@@ -1545,6 +1585,19 @@ impl FirecrackerBackend {
         if let Some(cache_root) = self.config.uffd_cache_root.as_ref() {
             cmd.arg("--cache-root").arg(cache_root);
         }
+        // ADR 0045 substrate (v2b): the handler creates + sizes the base
+        // shm file (it knows total_bytes from the canonical manifest)
+        // BEFORE binding the UDS, and `wait_for_socket` below orders the
+        // FC load after that — so FC's O_RDONLY open of the same path
+        // always sees a fully-sized file.
+        if let Some(base) = self.uffd_base_path(&canonical_ref) {
+            if let Some(dir) = base.parent() {
+                tokio::fs::create_dir_all(dir)
+                    .await
+                    .map_err(|e| vm_err(format!("create uffd base dir {}: {e}", dir.display())))?;
+            }
+            cmd.arg("--base-shm").arg(&base);
+        }
         if let Some(host) = prefault_trace_host {
             cmd.arg("--prefault-trace").arg(host.to_string());
         }
@@ -2550,13 +2603,20 @@ impl FirecrackerBackend {
                     .await?;
                 }
                 Some((_handler, uffd_uds)) => {
+                    // ADR 0045 substrate (v2b): same canonical ref as the
+                    // handler spawn (canonical == session, ADR 0015 M5),
+                    // so the derived base path matches the handler's.
+                    let base = manifest
+                        .memory_manifest
+                        .and_then(|r| self.uffd_base_path(&r));
                     tracing::Instrument::instrument(
                         async {
                             api.load_snapshot_uffd_opts(
                                 &state_path,
                                 uffd_uds,
                                 /*resume_vm=*/ aux_swap_plan.is_empty(),
                                 self.config.track_dirty_pages,
+                                base.as_deref(),
                             )
                             .await
                         },
@@ -4435,6 +4495,7 @@ mod tests {
             guest_otel_endpoint: None,
             firecracker_bin: PathBuf::from("/nonexistent/firecracker"),
             uffd_handler_bin: PathBuf::from("/nonexistent/engram-uffd-handler"),
+            uffd_base_dir: None,
             restore_mode: RestoreMode::File,
             base_restore_mode: None,
             track_dirty_pages: false,