fix(core): timeout due to node inactivity instead of total load test time (#2530)

Changes:

- Timeout is now smarter. Instead of waiting a fixed amount of time
(e.g. 10 minutes) for the whole load test to happen, which is a bit
unpredictable, the load test waits at most 1 minute (configurable) of
no-updates from the node. This way it's less machine dependent and more
based on responsiveness.
- load-test-ci.json is fixed to be similar to perf-ci.json, but in
prague and with the system smart contracts from l1-dev.json deployed.
- logs are re-added.
- Readme si fixed.
- Re-add flamegraph reporter to CI so they are generated on every push.

Closes #2522

Author: Arkenan

.github/scripts/flamegraph_watcher.sh

@@ -16,7 +16,7 @@ cargo build --release --manifest-path ./cmd/load_test/Cargo.toml
 
 echo "Starting load test"
 start_time=$(date +%s)
-RUST_BACKTRACE=1 ./target/release/load_test -k ./test_data/private_keys.txt -t eth-transfers -N 1000 -n http://localhost:1729 -w 5 >/dev/null
+RUST_BACKTRACE=1 ./target/release/load_test -k ./test_data/private_keys.txt -t eth-transfers -N 1000 -n http://localhost:1729 -w 1
 end_time=$(date +%s)
 
 elapsed=$((end_time - start_time))

.github/workflows/main_flamegraph_report.yaml

@@ -6,8 +6,8 @@ permissions:
   id-token: write
 
 on:
-  # push:
-  #   branches: ["main"]
+  push:
+    branches: ["main"]
   workflow_dispatch:
 
 env:
@@ -237,7 +237,7 @@ jobs:
           cargo build --release --bin ethrex --features dev
           CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph -c "record -o perf.data -F997 --call-graph dwarf,16384 -g" \
           --bin ethrex --release --features dev -- \
-          --dev --network /home/runner/work/ethrex/ethrex/test_data/genesis-perf-ci.json --http.port 1729 >/dev/null &
+          --dev --network /home/runner/work/ethrex/ethrex/test_data/genesis-load-test.json --http.port 1729 &
           sleep 10
           echo "Executing load test..."
           bash /home/runner/work/ethrex/ethrex/.github/scripts/flamegraph_watcher.sh &&
@@ -352,10 +352,10 @@ jobs:
           rm -rf target/profiling/reth
           cargo build --bin reth --profile profiling
           CARGO_PROFILE_RELEASE_DEBUG=true cargo flamegraph -c "record -o perf.data -F997 --call-graph dwarf,16384 -g" --bin reth --profile profiling -- \
-          node --chain /home/runner/work/ethrex/ethrex/test_data/genesis-perf-ci.json --dev \
+          node --chain /home/runner/work/ethrex/ethrex/test_data/genesis-load-test.json --dev \
           --dev.block-time 1000ms --http.port 1729  --txpool.max-pending-txns 100000000 --txpool.max-new-txns 1000000000 \
           --txpool.pending-max-count 100000000 --txpool.pending-max-size 10000000000 --txpool.basefee-max-count 100000000000 \
-          --txpool.basefee-max-size 1000000000000 --txpool.queued-max-count 1000000000 >/dev/null &
+          --txpool.basefee-max-size 1000000000000 --txpool.queued-max-count 1000000000 &
           sleep 30
           echo "Executing load test..."
           (cd /home/runner/work/ethrex/ethrex; ./.github/scripts/flamegraph_watcher.sh)

cmd/load_test/README.md

@@ -31,7 +31,7 @@ On some machines, this fixes the `ERROR axum::serve::listener: accept error: Too
 To run a load test, first run the node using a command like the following in the root folder:
 
 ```bash
-cargo run --bin ethrex --release --features dev -- --network test_data/genesis-perf-ci.json --dev
+cargo run --bin ethrex --release --features dev -- --network test_data/genesis-load-test.json --dev
 ```
 
 Genesis-l2-ci has many rich accounts and does not include the prague fork, which is important for dev mode until it's fixed.
@@ -73,39 +73,39 @@ Load tests are usually used to get performance metrics. We usually want to gener
 To produce a flamegraph, run the node in the following way.
 
 ```bash
-cargo flamegraph --root --bin ethrex --release --features dev -- --network test_data/genesis-perf-ci.json --dev
+cargo flamegraph --root --bin ethrex --release --features dev -- --network test_data/genesis-load-test.json --dev
 ```
 
 The "root" command is only needed for mac. It can be removed if running on linux.
 
 For a samply report, run the following:
 
 ```bash
-samply record cargo run --bin ethrex --release --features dev -- --network test_data/genesis-perf-ci.json --dev
+samply record cargo run --bin ethrex --release --features dev -- --network test_data/genesis-load-test.json --dev
 ```
 
 ## Interacting with reth
 
 The same load test can be run, the only difference is how you run the node:
 
 ```bash
-cargo run --release -- node --chain <path_to_ethrex>/test_data/genesis-perf-ci.json --dev --dev.block-time 5000ms --http.port 8545 --txpool.max-pending-txns 100000000 --txpool.max-new-txns 1000000000 --txpool.pending-max-count 100000000 --txpool.pending-max-size 10000000000 --txpool.basefee-max-count 100000000000 --txpool.basefee-max-size 1000000000000 --txpool.queued-max-count 1000000000
+cargo run --release -- node --chain <path_to_ethrex>/test_data/genesis-load-test.json --dev --dev.block-time 5000ms --http.port 8545 --txpool.max-pending-txns 100000000 --txpool.max-new-txns 1000000000 --txpool.pending-max-count 100000000 --txpool.pending-max-size 10000000000 --txpool.basefee-max-count 100000000000 --txpool.basefee-max-size 1000000000000 --txpool.queued-max-count 1000000000
 ```
 
 All of the txpool parameters are to make sure that it doesn't discard transactions sent by the load test. Trhoughput measurements in the logs are typically near 1Gigagas/second. To remove the database before getting measurements again:
 
 ```bash
-cargo run --release -- db --chain <path_to_ethrex>/test_data/genesis-perf-ci.json drop -f
+cargo run --release -- db --chain <path_to_ethrex>/test_data/genesis-load-test.json drop -f
 ```
 
 To get a flamegraph of its execution, run with the same parameters, just replace `cargo run --release` with `cargo flamegraph --bin reth --profiling`:
 
 ```bash
-cargo flamegraph --bin reth --root --profiling -- node --chain ~/workspace/ethrex/test_data/genesis-perf-ci.json --dev --dev.block-time 5000ms --http.port 8545 --txpool.max-pending-txns 100000000 --txpool.max-new-txns 1000000000 --txpool.pending-max-count 100000000 --txpool.pending-max-size 10000000000 --txpool.basefee-max-count 100000000000 --txpool.basefee-max-size 1000000000000 --txpool.queued-max-count 1000000000
+cargo flamegraph --bin reth --root --profiling -- node --chain ~/workspace/ethrex/test_data/genesis-load-test.json --dev --dev.block-time 5000ms --http.port 8545 --txpool.max-pending-txns 100000000 --txpool.max-new-txns 1000000000 --txpool.pending-max-count 100000000 --txpool.pending-max-size 10000000000 --txpool.basefee-max-count 100000000000 --txpool.basefee-max-size 1000000000000 --txpool.queued-max-count 1000000000
 ```
 
 For samply we want to directly execute the binary, so that it records the binary and not cargo itself:
 
 ```bash
-samply record ./target/profiling/reth node --chain ~/workspace/ethrex/test_data/genesis-perf-ci.json --dev --dev.block-time 5000ms --http.port 8545 --txpool.max-pending-txns 100000000 --txpool.max-new-txns 1000000000 --txpool.pending-max-count 100000000 --txpool.pending-max-size 10000000000 --txpool.basefee-max-count 100000000000 --txpool.basefee-max-size 1000000000000 --txpool.queued-max-count 1000000000
+samply record ./target/profiling/reth node --chain ~/workspace/ethrex/test_data/genesis-load-test.json --dev --dev.block-time 5000ms --http.port 8545 --txpool.max-pending-txns 100000000 --txpool.max-new-txns 1000000000 --txpool.pending-max-count 100000000 --txpool.pending-max-size 10000000000 --txpool.basefee-max-count 100000000000 --txpool.basefee-max-size 1000000000000 --txpool.queued-max-count 1000000000
 ```

cmd/load_test/src/main.rs

@@ -62,7 +62,7 @@ struct Cli {
         long,
         short = 'w',
         default_value_t = 0,
-        help = "Timeout to wait for all transactions to be included. If 0 is specified, wait indefinitely."
+        help = "Timeout in minutes. If the node doesn't provide updates in this time, it's considered stuck and the load test fails. If 0 is specified, the load test will wait indefinitely."
     )]
     wait: u64,
 }
@@ -248,13 +248,11 @@ async fn load_test(
                 let client = client.clone();
                 sleep(Duration::from_micros(800)).await;
                 let _sent = client.send_eip1559_transaction(&tx, &sk).await?;
-                println!(
-                    "Tx number {} sent! From: {}. To: {}",
-                    nonce + i + 1,
-                    encoded_src,
-                    dst.encode_hex::<String>()
-                );
             }
+            println!(
+                "{} transactions have been sent for {}",
+                tx_amount, encoded_src
+            );
             Ok::<(), EthClientError>(())
         });
     }
@@ -268,18 +266,18 @@ async fn load_test(
 // Waits until the nonce of each account has reached the tx_amount.
 async fn wait_until_all_included(
     client: EthClient,
-    wait: Option<Duration>,
+    timeout: Option<Duration>,
     accounts: &[Account],
     tx_amount: u64,
 ) -> Result<(), String> {
-    let start_time = tokio::time::Instant::now();
-
     for (_, sk) in accounts {
         let client = client.clone();
         let src = get_address_from_secret_key(sk).expect("Failed to get address from secret key");
         let encoded_src: String = src.encode_hex();
+        let mut last_updated = tokio::time::Instant::now();
+        let mut last_nonce = 0;
+
         loop {
-            let elapsed = start_time.elapsed();
             let nonce = client.get_nonce(src, BlockByNumber::Latest).await.unwrap();
             if nonce >= tx_amount {
                 println!(
@@ -289,14 +287,23 @@ async fn wait_until_all_included(
                 break;
             } else {
                 println!(
-                    "Waiting for transactions to be included from {}. Nonce: {}. Needs: {}. Percentage: {:2}%. Elapsed time: {}s.",
-                    encoded_src, nonce, tx_amount, (nonce as f64 / tx_amount as f64) * 100.0, elapsed.as_secs()
+                    "Waiting for transactions to be included from {}. Nonce: {}. Needs: {}. Percentage: {:2}%.",
+                    encoded_src, nonce, tx_amount, (nonce as f64 / tx_amount as f64) * 100.0
                 );
             }
 
-            if let Some(wait) = wait {
-                if elapsed > wait {
-                    return Err("Timeout reached for transactions to be included".to_string());
+            if let Some(timeout) = timeout {
+                if last_nonce == nonce {
+                    let inactivity_time = last_updated.elapsed();
+                    if inactivity_time > timeout {
+                        return Err(format!(
+                            "Node inactive for {} seconds. Timeout reached.",
+                            inactivity_time.as_secs()
+                        ));
+                    }
+                } else {
+                    last_nonce = nonce;
+                    last_updated = tokio::time::Instant::now();
                 }
             }
 
@@ -379,7 +386,7 @@ async fn main() {
     };
 
     println!(
-        "Starting load test with {} transactions per account",
+        "Starting load test with {} transactions per account...",
         cli.tx_amount
     );
     let time_now = tokio::time::Instant::now();