Visualizer flexibility (#237)

* sim-rs: select a scenario to visualize

* sim-rs: move stream processing logic out of server

* sim-rs: handle stream processing in web worker

* sim-rs: fix cbor/json encoding

* sim-rs: always report when streaming is done

* sim-rs: update documentation

Author: SupernaviX

sim-rs/Cargo.lock

@@ -12,12 +12,12 @@ clap = { version = "4", features = ["derive"] }
 ctrlc = "3"
 figment = { version = "0.10", features = ["yaml"] }
 itertools = "0.14"
+minicbor-serde = { version = "0.3", features = ["alloc"] }
 netsim-core = { git = "https://github.com/input-output-hk/ce-netsim", rev = "9d1e26c" }
 pretty-bytes-rust = "0.3.0"
 rand = "0.9"
 statrs = "0.18"
 serde = { version = "1", features = ["derive"] }
-serde_cbor = "0.11"
 serde_json = "1"
 serde_yaml = "0.9"
 sim-core = { path = "../sim-core" }

sim-rs/sim-cli/Cargo.toml

@@ -573,7 +573,7 @@ impl OutputTarget {
                 file.write_all(string.as_bytes()).await?;
             }
             OutputFormat::CborStream => {
-                let bytes = serde_cbor::to_vec(&event)?;
+                let bytes = minicbor_serde::to_vec(&event)?;
                 file.write_all(&bytes).await?;
             }
         }

sim-rs/sim-cli/src/events.rs

@@ -280,7 +280,11 @@ impl EventTracker {
             endorsement: block.endorsement.as_ref().map(|e| Endorsement {
                 eb: self.to_endorser_block(e.eb),
                 bytes: e.bytes,
-                votes: e.votes.iter().map(|n| self.to_node(*n)).collect(),
+                votes: e
+                    .votes
+                    .iter()
+                    .map(|(k, v)| (self.to_node(*k), *v))
+                    .collect(),
             }),
             transactions: block.transactions.iter().map(|tx| tx.id).collect(),
         });

sim-rs/sim-core/src/events.rs

@@ -138,7 +138,7 @@ impl<Node: Display + Serialize> Serialize for EndorserBlockId<Node> {
     where
         S: serde::Serializer,
     {
-        let mut obj = serializer.serialize_struct("EndorserBlockId", 4)?;
+        let mut obj = serializer.serialize_struct("EndorserBlockId", 3)?;
         obj.serialize_field("id", &self.to_string())?;
         obj.serialize_field("slot", &self.slot)?;
         obj.serialize_field("producer", &self.producer)?;
@@ -178,7 +178,7 @@ impl<Node: Display + Serialize> Serialize for VoteBundleId<Node> {
     where
         S: serde::Serializer,
     {
-        let mut obj = serializer.serialize_struct("VoteBundleId", 4)?;
+        let mut obj = serializer.serialize_struct("VoteBundleId", 3)?;
         obj.serialize_field("id", &self.to_string())?;
         obj.serialize_field("slot", &self.slot)?;
         obj.serialize_field("producer", &self.producer)?;
@@ -204,5 +204,5 @@ pub enum NoVoteReason {
 pub struct Endorsement<Node: Display = NodeId> {
     pub eb: EndorserBlockId<Node>,
     pub bytes: u64,
-    pub votes: Vec<Node>,
+    pub votes: BTreeMap<Node, usize>,
 }

sim-rs/sim-core/src/model.rs

@@ -138,7 +138,7 @@ struct NodeLeiosState {
     ebs: BTreeMap<EndorserBlockId, EndorserBlockState>,
     ebs_by_slot: BTreeMap<u64, Vec<EndorserBlockId>>,
     votes_to_generate: BTreeMap<u64, usize>,
-    votes_by_eb: BTreeMap<EndorserBlockId, Vec<NodeId>>,
+    votes_by_eb: BTreeMap<EndorserBlockId, BTreeMap<NodeId, usize>>,
     votes: BTreeMap<VoteBundleId, VoteBundleState>,
 }
 
@@ -311,12 +311,7 @@ impl Node {
             CpuTask::RBBlockGenerated(block) => {
                 let mut time = cpu_times.rb_generation;
                 if let Some(endorsement) = &block.endorsement {
-                    let nodes = endorsement
-                        .votes
-                        .iter()
-                        .copied()
-                        .collect::<HashSet<_>>()
-                        .len();
+                    let nodes = endorsement.votes.len();
                     time += cpu_times.cert_generation_constant
                         + (cpu_times.cert_generation_per_node * nodes as u32)
                 }
@@ -327,12 +322,7 @@ impl Node {
                 let bytes: u64 = rb.transactions.iter().map(|tx| tx.bytes).sum();
                 time += cpu_times.rb_validation_per_byte * (bytes as u32);
                 if let Some(endorsement) = &rb.endorsement {
-                    let nodes = endorsement
-                        .votes
-                        .iter()
-                        .copied()
-                        .collect::<HashSet<_>>()
-                        .len();
+                    let nodes = endorsement.votes.len();
                     time += cpu_times.cert_validation_constant
                         + (cpu_times.cert_validation_per_node * nodes as u32)
                 }
@@ -732,14 +722,17 @@ impl Node {
             .leios
             .votes_by_eb
             .iter()
-            .filter(|(eb, votes)| {
-                votes.len() as u64 >= vote_threshold && !forbidden_slots.contains(&eb.slot)
+            .filter_map(|(eb, votes)| {
+                let vote_count: usize = votes.values().sum();
+                if (vote_count as u64) < vote_threshold || forbidden_slots.contains(&eb.slot) {
+                    return None;
+                }
+                Some((eb, vote_count))
             })
-            .max_by_key(|(eb, votes)| (self.count_txs_in_eb(eb), votes.len()))?;
+            .max_by_key(|(eb, votes)| (self.count_txs_in_eb(eb), *votes))?;
 
         let (block, votes) = self.leios.votes_by_eb.remove_entry(&block)?;
-        let nodes = votes.iter().collect::<HashSet<_>>().len();
-        let bytes = self.sim_config.sizes.cert(nodes);
+        let bytes = self.sim_config.sizes.cert(votes.len());
 
         Some(Endorsement {
             eb: block,
@@ -1120,11 +1113,13 @@ impl Node {
             return Ok(());
         }
         for (eb, count) in votes.ebs.iter() {
-            self.leios
+            *self
+                .leios
                 .votes_by_eb
                 .entry(*eb)
                 .or_default()
-                .extend(std::iter::repeat(votes.id.producer).take(*count));
+                .entry(votes.id.producer)
+                .or_default() += count;
         }
         // We haven't seen these votes before, so propagate them to our neighbors
         for peer in &self.consumers {
@@ -1264,11 +1259,13 @@ impl Node {
     fn finish_generating_vote_bundle(&mut self, votes: VoteBundle) -> Result<()> {
         self.tracker.track_votes_generated(&votes);
         for (eb, count) in &votes.ebs {
-            self.leios
+            *self
+                .leios
                 .votes_by_eb
                 .entry(*eb)
                 .or_default()
-                .extend(std::iter::repeat(votes.id.producer).take(*count));
+                .entry(votes.id.producer)
+                .or_default() += count;
         }
         let votes = Arc::new(votes);
         self.leios

sim-rs/sim-core/src/sim/node.rs

@@ -128,3 +128,8 @@ dist
 .yarn/build-state.yml
 .yarn/install-state.gz
 .pnp.*
+
+# don't commit these large files until we're ready
+public/scenarios.json
+public/topologies
+public/traces

ui/.gitignore

@@ -18,14 +18,34 @@ Before you can run the UI, ensure you have the following installed:
 
 2. **Generate a Trace**
 
-   Before running the UI, you need to generate a trace using the `sim-rs`
-   simulation. Navigate to the `sim-rs` directory and run the following command:
+   Before running the UI, you need to prepare a scenario for it to visualize. Add the following to a file named `public/scenarios.json`:
+
+   ```json
+   {
+      "scenarios": [
+         {
+            "name": "My Scenario",
+            "topology": "topologies/thousand.yaml",
+            "duration": 30.0,
+            "trace": "traces/myscenario.jsonl"
+         }
+      ]
+   }
+   ```
 
-   ```bash
-   cargo run --release test_data/thousand.yaml output/messages.jsonl -s 30
+   Now add that topology to the public directory:
+
+   ```sh
+   mkdir -p public/topologies
+   cp ../sim-rs/test-data/thousand.yaml public/topologies
    ```
 
-   This command will generate a trace file at `output/messages.jsonl`.
+   And generate a trace to visualize from the `sim-rs` directory:
+
+   ```bash
+   mkdir -p ../ui/public/traces
+   cargo run --release test_data/thousand.yaml ../ui/public/traces/myscenario.jsonl -s 30
+   ```
 
 3. **Run the UI**
 

ui/README.md

@@ -59,16 +59,11 @@ ui-app: {
 
   backend: {
     class: component
-    label: "Backend API\n- Data Fetching\n- Trace Processing"
+    label: "Static Server"
 
-    api_routes: {
+    static_files: {
       class: subcomponent
-      label: "API Routes\n- /api/messages\n- /api/trace"
-    }
-
-    data_processing: {
-      class: subcomponent
-      label: "Data Processing\n- JSON Parsing\n- Data Transformation"
+      label: "Static React web page\nSimulation traces"
     }
   }
 }
@@ -87,4 +82,3 @@ external-systems: {
 ui-app.frontend -> ui-app.backend: "Fetches data from"
 ui-app.backend -> external-systems.sim-rs: "Reads trace data from"
 ui-app.frontend.components -> ui-app.frontend.state: "Uses for state management"
-ui-app.backend.api_routes -> ui-app.backend.data_processing: "Processes data for"

ui/docs/component.d2

@@ -49,7 +49,7 @@ ui-web-app: {
 
   server: {
     class: component
-    label: "Backend Server\n- API Endpoints\n- Data Processing"
+    label: "Static backend server"
   }
 }
 
@@ -70,5 +70,5 @@ external-systems: {
 
 # Relationships
 ui-web-app.browser -> ui-web-app.server: "Requests data from"
-ui-web-app.server -> external-systems.trace-file: "Reads data from"
+ui-web-app.server -> external-systems.trace-file: "Serves data from"
 external-systems.sim-rs -> external-systems.trace-file: "Generates"