feat: add benchmark for trin

Author: KolbyML

.circleci/config.yml

@@ -235,7 +235,7 @@ jobs:
     # parallelism level should be set to the amount of simulators we have or greater
     # The reason for this is the CI code currently only supports 1 input at a time
     # if we have a parallelism level of 5 and 6 sims one test runner will get 2 test sims and fail
-    parallelism: 19
+    parallelism: 20
     steps:
       - checkout
       - run:

.gitignore

@@ -7,4 +7,4 @@
 .vscode/
 # Ignore downloaded consensus specs test data
 testing/ef-tests/mainnet*
-
+bin/trin-bench/logs*

Cargo.lock

@@ -1,7 +1,8 @@
 [workspace]
 members = [
     "bin/portal-bridge",
-    "bin/trin", 
+    "bin/trin",
+    "bin/trin-bench",
     "bin/trin-execution",
     "crates/ethportal-api",
     "crates/e2store",
@@ -15,7 +16,7 @@ members = [
     "crates/subnetworks/history",
     "crates/subnetworks/state",
     "crates/utils",
-    "crates/validation",    
+    "crates/validation",
     "testing/ef-tests",
     "testing/ethportal-peertest",
     "testing/utp",
@@ -120,3 +121,8 @@ trin-validation = { path = "crates/validation" }
 #
 # See: https://github.com/eira-fransham/crunchy/issues/13
 crunchy = "=0.2.2"
+
+
+[profile.profiling]
+inherits = "release"
+debug = true

Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "trin-bench"
+authors.workspace = true
+categories.workspace = true
+edition.workspace = true
+keywords.workspace = true
+license.workspace = true
+readme.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+clap.workspace = true
+ethportal-api.workspace = true
+e2store.workspace = true
+futures.workspace = true
+humanize-duration.workspace = true
+jsonrpsee.workspace = true
+portal-bridge.workspace = true
+reqwest.workspace = true
+tokio.workspace = true
+tracing.workspace = true
+tracing-subscriber.workspace = true
+trin-execution.workspace = true
+trin-utils.workspace = true
+trin-validation.workspace = true
+url.workspace = true

bin/trin-bench/Cargo.toml

@@ -0,0 +1,12 @@
+bench-trin:
+	@echo "Running Trin Bench"
+	@./run-bench.sh trin
+	@echo "Down running Trin Bench"
+
+bench-trin-perf:
+	@echo "Running Trin Bench"
+	@./run-bench.sh trin perf
+	@echo "Down running Trin Bench"	
+
+clean:
+	sudo rm -r logs

bin/trin-bench/Makefile

@@ -0,0 +1,29 @@
+# Trin Bench
+Trin bench is for testing Trin performance and being able to recreate situations reliably to verify if performance improvements or regressions happen
+
+## To run benchmark with Trin
+```sh
+make bench-trin
+```
+
+## To run benchmark with Trin and generate flamegraphs
+```sh
+make bench-trin-perf
+```
+
+## Clean results
+```sh
+make clean
+```
+
+## View the results
+
+The results such as logs can be viewed in the generate `logs` folder
+
+`trin_benchmark.log` will contain the logs of the bench mark coordinator
+
+`data_sender` and `data_receiver` is the data directory
+`data_sender.log` and `data_receiver.log` are the logs of the portal clients
+if perf mode is ran the portal client will be profiled and flamegraphs will be generated `data_sender.svg` and `data_receiver.svg`
+
+The `past_runs` directory archives the logs and flamegraphs of previous runs

bin/trin-bench/README.md

@@ -0,0 +1,168 @@
+#!/bin/bash
+
+# Define log directories
+LOG_DIR="logs"
+PAST_RUNS_DIR="$LOG_DIR/past_runs"
+mkdir -p "$PAST_RUNS_DIR"
+
+# Create data directories for portal client instances
+DATA_DIR_SENDER="data_sender"
+DATA_DIR_RECEIVER="data_receiver"
+mkdir -p "$LOG_DIR/$DATA_DIR_SENDER" "$LOG_DIR/$DATA_DIR_RECEIVER"
+
+# Check first argument: portal client selection
+PORTAL_CLIENT="$1"
+VALID_CLIENTS=("trin" "fluffy" "shisui" "ultralight" "samba")
+if [[ ! " ${VALID_CLIENTS[@]} " =~ " $PORTAL_CLIENT " ]]; then
+    echo "Error: Invalid portal client specified. Choose from: ${VALID_CLIENTS[*]}"
+    exit 1
+fi
+
+# Check second argument: perf flag
+USE_PERF=false
+if [ "$2" == "perf" ]; then
+    USE_PERF=true
+fi
+
+# Clone portal-accumulators repository if not already present
+if [ ! -d "../../portal-accumulators" ]; then
+    git clone https://github.com/ethereum/portal-accumulators ../../portal-accumulators || { echo "Failed to clone portal-accumulators"; exit 1; }
+fi
+
+# Build trin-benchmark-coordinator with release profile
+pushd ../.. || { echo "Failed to change directory"; exit 1; }
+cargo build --release -p trin-bench || { echo "Failed to build trin-benchmark-coordinator"; exit 1; }
+popd || { echo "Failed to return to original directory"; exit 1; }
+
+if [ "$PORTAL_CLIENT" == "trin" ]; then
+    pushd ../.. || { echo "Failed to change directory"; exit 1; }
+    cargo build --release -p trin || { echo "Failed to build trin"; exit 1; }
+    popd || { echo "Failed to return to original directory"; exit 1; }
+fi
+
+# Define process PIDs
+PIDS=()
+
+# Find available ports dynamically and ensure they are unique
+find_unused_port() {
+    local port=$1
+    while ss -tuln | awk '{print $4}' | grep -q ":$port$"; do
+        port=$((port + 1))
+    done
+    echo $port
+}
+
+PORT_SENDER=$(find_unused_port 9050)
+PORT_RECEIVER=$(find_unused_port $((PORT_SENDER + 10)))
+EXT_PORT_SENDER=$(find_unused_port 9100)
+EXT_PORT_RECEIVER=$(find_unused_port $((EXT_PORT_SENDER + 10)))
+
+# Check if perf flag is passed
+USE_PERF=false
+if [ "$1" == "perf" ]; then
+    USE_PERF=true
+fi
+
+run_trin() {
+    local log_file="$1"
+    local web3_address="$2"
+    local external_address="$3"
+    local discovery_port="$4"
+    local data_dir="$5"
+    local mb="$6"
+    
+    if $USE_PERF; then
+        cargo flamegraph --profile release -c "record -F 97 --call-graph dwarf,64000 -g -o $log_file.perf" --release --output "$log_file.svg" -p trin -- \
+            --web3-transport http \
+            --web3-http-address "$web3_address" \
+            --mb "$mb" \
+            --bootnodes none \
+            --external-address "$external_address" \
+            --discovery-port "$discovery_port" \
+            --data-dir "$data_dir" \
+            --max-radius 100 \            
+            > "$log_file.log" 2>&1 &
+    else
+        ../../target/release/trin \
+            --web3-transport http \
+            --web3-http-address "$web3_address" \
+            --mb "$mb" \
+            --bootnodes none \
+            --external-address "$external_address" \
+            --discovery-port "$discovery_port" \
+            --data-dir "$data_dir" \
+            --max-radius 100 \
+            > "$log_file.log" 2>&1 &
+    fi
+    PIDS+=("$!")
+}
+
+if [ "$PORTAL_CLIENT" == "trin" ]; then
+    # Run trin sender
+    run_trin "$LOG_DIR/$DATA_DIR_SENDER" "http://127.0.0.1:$PORT_SENDER/" "127.0.0.1:$EXT_PORT_SENDER" "$EXT_PORT_SENDER" "$LOG_DIR/$DATA_DIR_SENDER" "0"
+
+    # Run trin receiver
+    run_trin "$LOG_DIR/$DATA_DIR_RECEIVER" "http://127.0.0.1:$PORT_RECEIVER/" "127.0.0.1:$EXT_PORT_RECEIVER" "$EXT_PORT_RECEIVER" "$LOG_DIR/$DATA_DIR_RECEIVER" "10000"
+fi
+
+# Run trin benchmark coordinator
+../../target/release/trin-bench \
+    --web3-http-address-node-1 http://127.0.0.1:$PORT_SENDER/ \
+    --web3-http-address-node-2 http://127.0.0.1:$PORT_RECEIVER/ \
+    --epoch-accumulator-path ../../portal-accumulators \
+    --start-era1 1000 \
+    --end-era1 1010 \
+    > "$LOG_DIR/trin_benchmark.log" 2>&1 &
+TRIN_BENCH_PID=$!
+
+echo "Started Benchmark"
+
+CLEANED_UP=false
+cleanup() {
+    if $CLEANED_UP; then
+        return
+    fi
+    CLEANED_UP=true
+    echo "Finished benchmark. Stopping processes..."
+    
+    for PID in "${PIDS[@]}"; do
+        if kill -0 "$PID" 2>/dev/null; then
+            echo "Killing process with PID $PID..."
+            kill -SIGINT "$PID"
+            pkill -SIGINT -P "$PID"
+        fi
+    done
+    
+    for PID in "${PIDS[@]}"; do
+        if kill -0 "$PID" 2>/dev/null; then
+            echo "Waiting process with PID $PID..."
+            wait "$PID" 2>/dev/null
+        fi
+    done
+    
+    if kill -0 "$TRIN_BENCH_PID" 2>/dev/null; then
+        echo "Stopping trin-bench with PID $TRIN_BENCH_PID..."
+        kill -SIGINT "$TRIN_BENCH_PID"
+        wait "$TRIN_BENCH_PID" 2>/dev/null
+    fi
+
+    echo "All processes stopped."
+    sudo rm -rf "$LOG_DIR/$DATA_DIR_SENDER" "$LOG_DIR/$DATA_DIR_RECEIVER" "$LOG_DIR/$DATA_DIR_SENDER.perf" "$LOG_DIR/$DATA_DIR_RECEIVER.perf"
+
+    # Generate timestamp-based folder name
+    TIMESTAMP=$(date +%s)
+    PERF_TAG=$([ "$USE_PERF" == true ] && echo "_perf" || echo "")
+    RUN_FOLDER="$PAST_RUNS_DIR/${TIMESTAMP}_${PORTAL_CLIENT}${PERF_TAG}"
+    mkdir -p "$RUN_FOLDER"
+
+    # Move logs and performance files to the archive folder
+    find "$LOG_DIR" -maxdepth 1 -type f -name "*.log" -exec mv {} "$RUN_FOLDER/" \;
+    find "$LOG_DIR" -maxdepth 1 -type f -name "*.svg" -exec mv {} "$RUN_FOLDER/" \;
+
+    echo "Archived logs to $RUN_FOLDER"
+}
+
+trap cleanup SIGINT SIGTERM ERR
+wait "$TRIN_BENCH_PID"
+trap - SIGINT SIGTERM ERR
+cleanup

bin/trin-bench/run-bench.sh

@@ -0,0 +1,50 @@
+use std::path::PathBuf;
+
+use clap::Parser;
+use url::Url;
+
+pub const APP_NAME: &str = "trin-bench";
+const DEFAULT_EPOCH_ACC_PATH: &str = "./portal-accumulators";
+
+#[derive(Parser, Debug, Clone)]
+#[command(
+    name = "Trin Bench",
+    about = "Benchmarks sending blocks from one trin client to another"
+)]
+pub struct TrinBenchConfig {
+    #[arg(
+        long = "web3-http-address-node-1",
+        help = "address to accept json-rpc http connections"
+    )]
+    pub web3_http_address_node_1: Url,
+
+    #[arg(
+        long = "web3-http-address-node-2",
+        help = "address to accept json-rpc http connections"
+    )]
+    pub web3_http_address_node_2: Url,
+
+    #[arg(
+        long,
+        help = "The first era to start sending blocks from",
+        default_value = "1"
+    )]
+    pub start_era1: u16,
+
+    #[arg(long, help = "The last era to send blocks from", default_value = "5")]
+    pub end_era1: u16,
+
+    #[arg(
+        long = "epoch-accumulator-path",
+        help = "Path to epoch accumulator repo for bridge mode",
+        default_value = DEFAULT_EPOCH_ACC_PATH
+    )]
+    pub epoch_acc_path: PathBuf,
+
+    #[arg(
+        long,
+        help = "The max amount of offer requests to have open at any given time",
+        default_value = "10"
+    )]
+    pub offer_concurrency: usize,
+}

bin/trin-bench/src/cli.rs

@@ -0,0 +1 @@
+pub mod cli;