flowey: fix saving secret vars (#1417) (#1421)

We are trying to write a non-JSON-formatted value to track that an
env-var-sourced variable is secret. Fix this by writing `null`. Also add
in some diagnostics and improve the in-memory variable representation to
avoid so many allocations.

Co-authored-by: John Starks <[email protected]>

Author: benhillis

flowey/flowey_cli/Cargo.toml

@@ -19,7 +19,7 @@ log.workspace = true
 parking_lot.workspace = true
 petgraph.workspace = true
 serde = { workspace = true, features = ["derive"] }
-serde_json.workspace = true
+serde_json = { workspace = true, features = ["raw_value"] }
 serde_yaml.workspace = true
 toml_edit = { workspace = true, features = ["serde"] }
 xshell.workspace = true

flowey/flowey_cli/src/cli/var_db.rs

@@ -273,7 +273,7 @@ impl VarDb {
                 if is_secret {
                     // Remember that this environment variable is secret so that
                     // it cannot be easily laundered into a non-secret variable.
-                    runtime_var_db.set_var(&env_source_name(&env), false, Vec::new());
+                    runtime_var_db.set_var(&env_source_name(&env), false, "null".into());
                 }
 
                 match backend {

flowey/flowey_cli/src/var_db/single_json_file.rs

@@ -7,6 +7,8 @@ use anyhow::Context;
 use fs_err::File;
 use serde::Deserialize;
 use serde::Serialize;
+use serde_json::value::RawValue;
+use std::borrow::Cow;
 use std::collections::BTreeMap;
 use std::io::Seek;
 use std::io::Write;
@@ -15,8 +17,8 @@ use std::path::Path;
 /// On-disk format for the var db
 #[derive(Serialize, Deserialize)]
 #[serde(transparent)]
-struct VarDb {
-    vars: BTreeMap<String, (bool, serde_json::Value)>,
+struct VarDb<'a> {
+    vars: BTreeMap<String, (bool, Cow<'a, RawValue>)>,
 }
 
 /// Implements [`flowey_core::node::RuntimeVarDb`] backed by a JSON file.
@@ -47,7 +49,7 @@ impl SingleJsonFileVarDb {
         Ok(Self { file })
     }
 
-    fn load_db(&mut self) -> VarDb {
+    fn load_db(&mut self) -> VarDb<'static> {
         self.file.rewind().unwrap();
         serde_json::from_reader(&self.file).expect("corrupt runtime variable db")
     }
@@ -57,30 +59,26 @@ impl flowey_core::node::RuntimeVarDb for SingleJsonFileVarDb {
     fn try_get_var(&mut self, var_name: &str) -> Option<(Vec<u8>, bool)> {
         let db = self.load_db();
         let (is_secret, ref val) = *db.vars.get(var_name)?;
-        let val = val.to_string();
         if is_secret {
             log::debug!("[db] read var: {} = <secret>", var_name);
         } else {
             log::debug!("[db] read var: {} = {}", var_name, val);
         }
-        Some((val.into(), is_secret))
+        Some((val.get().into(), is_secret))
     }
 
     fn set_var(&mut self, var_name: &str, is_secret: bool, value: Vec<u8>) {
+        let value: &RawValue = serde_json::from_slice(&value)
+            .unwrap_or_else(|err| panic!("invalid JSON for var {}: {}", var_name, err));
         if is_secret {
             log::debug!("[db] set var: {} = <secret>", var_name)
         } else {
-            log::debug!(
-                "[db] set var: {} = {}",
-                var_name,
-                String::from_utf8_lossy(&value)
-            )
+            log::debug!("[db] set var: {} = {}", var_name, value)
         };
         let mut db = self.load_db();
-        let existing = db.vars.insert(
-            var_name.into(),
-            (is_secret, serde_json::from_slice(&value).unwrap()),
-        );
+        let existing = db
+            .vars
+            .insert(var_name.into(), (is_secret, Cow::Borrowed(value)));
         assert!(existing.is_none()); // all vars are one-time-write
         self.file.set_len(0).unwrap();
         self.file.rewind().unwrap();