feat: add test_scanner

Cargo.toml

@@ -58,19 +58,25 @@ openvm-transpiler = { git = "https://github.com/openvm-org/openvm.git", tag = "v
 openvm-stark-sdk = { git = "https://github.com/openvm-org/stark-backend.git", rev = "f48090c9febd021f8ee0349bc929a775fb1fa3ad" }
 p3-field = { git = "https://github.com/Plonky3/Plonky3.git", rev = "539bbc84085efb609f4f62cb03cf49588388abdb" }
 
-sbv-core = { git = "https://github.com/scroll-tech/stateless-block-verifier", branch = "chore/upgrade", features = ["scroll"] }
-sbv-primitives = { git = "https://github.com/scroll-tech/stateless-block-verifier", branch = "chore/upgrade", features = ["scroll", "serde", "rkyv"] }
-sbv-kv = { git = "https://github.com/scroll-tech/stateless-block-verifier", branch = "chore/upgrade" }
-sbv-trie = { git = "https://github.com/scroll-tech/stateless-block-verifier", branch = "chore/upgrade" }
+sbv-core = { git = "https://github.com/scroll-tech/stateless-block-verifier", branch = "master", features = ["scroll"] }
+sbv-primitives = { git = "https://github.com/scroll-tech/stateless-block-verifier", branch = "master", features = ["scroll", "serde", "rkyv"] }
+sbv-utils = { git = "https://github.com/scroll-tech/stateless-block-verifier", branch = "master", features = ["scroll"] }
+sbv-kv = { git = "https://github.com/scroll-tech/stateless-block-verifier", branch = "master" }
+sbv-trie = { git = "https://github.com/scroll-tech/stateless-block-verifier", branch = "master" }
 
+alloy-provider = "1.0"
+alloy-rpc-client = "1.0"
 alloy-primitives = { version = "1.2", default-features = false, features = ["std", "map-hashbrown", "map-fxhash", "rkyv"] }
 alloy-serde = { version = "1.0", default-features = false }
+alloy-transport = "=1.0.12"
 bitcode = { version = "0.6.5", default-features = false, features = ["serde", "derive"] }
 bincode_v1 = { version = "1.3", package = "bincode"}
 bincode = { version = "2.0" }
+cargo_metadata = "0.20"
 c-kzg = { version = "2.0" }
 derivative = "2.2.0"
 eyre = "0.6"
+futures = "0.3"
 halo2curves-axiom = { version = "0.5.3" }
 itertools = "0.14"
 hex-literal = { version = "0.4.1", default-features = false }
@@ -90,11 +96,14 @@ snark-verifier-sdk = { version = "=0.2.1", default-features = false, features =
     "display",
     "revm",
 ] }
+sysinfo = { version = "0.35", default-features = false }
 tiny-keccak = "2.0"
 tracing = "0.1"
+tokio = "1"
 vm-zstd = { git = "https://github.com/scroll-tech/rust-zstd-decompressor.git", branch = "master" }
 toml = "0.8.14"
 tracing-subscriber = "0.3"
+url = "2.5"
 
 scroll-zkvm-types = { path = "crates/types" }
 scroll-zkvm-types-base = { path = "crates/types/base" }

crates/integration/Cargo.toml

@@ -8,6 +8,7 @@ scroll-zkvm-types.workspace = true
 scroll-zkvm-prover.workspace = true
 scroll-zkvm-verifier.workspace = true
 sbv-primitives = { workspace = true }
+sbv-utils.workspace = true
 tracing.workspace = true
 rkyv.workspace = true
 tracing-subscriber.workspace = true
@@ -22,10 +23,18 @@ openvm-native-recursion = { workspace = true, default-features = false }
 openvm-native-transpiler = { workspace = true, default-features = false }
 openvm-transpiler = { workspace = true, default-features = false }
 
+alloy-provider = { workspace = true }
+alloy-rpc-client.workspace = true
+alloy-transport = { workspace = true, features = ["throttle"] }
 alloy-primitives.workspace = true
+cargo_metadata.workspace = true
 eyre.workspace = true
+futures.workspace = true
 rayon.workspace = true
 vm-zstd = { workspace = true, features = ["zstd"] }
+sysinfo = { workspace = true, features = ["system"] }
+tokio = { workspace = true, features = ["full", "macros"] }
+url.workspace = true
 
 chrono = "0.4"
 ff = "0.13"

crates/integration/src/lib.rs

@@ -1,8 +1,4 @@
-use std::{
-    path::{Path, PathBuf},
-    process,
-};
-
+use cargo_metadata::MetadataCommand;
 use once_cell::sync::OnceCell;
 use openvm_sdk::{
     F, Sdk,
@@ -13,15 +9,36 @@ use scroll_zkvm_prover::{
     setup::{read_app_config, read_app_exe},
     task::ProvingTask,
 };
+use std::{
+    path::{Path, PathBuf},
+    process,
+    sync::LazyLock,
+};
 use tracing::instrument;
 use tracing_subscriber::{fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt};
 
 pub mod testers;
 
 pub mod utils;
 
+pub static WORKSPACE_ROOT: LazyLock<&Path> = LazyLock::new(|| {
+    let path = MetadataCommand::new()
+        .no_deps()
+        .exec()
+        .expect("failed to execute cargo-metadata")
+        .workspace_root
+        .into_std_path_buf();
+    eprintln!("WORKSPACE_ROOT = {}", path.display());
+    Box::leak(path.into_boxed_path())
+});
+
 /// Path to store release assets, root directory of zkvm-prover repository.
-const DIR_OUTPUT: &str = "./../../.output";
+static DIR_OUTPUT: LazyLock<&Path> = LazyLock::new(|| {
+    let path = WORKSPACE_ROOT.join(".output");
+    std::fs::create_dir_all(&path).expect("failed to create output directory");
+    eprintln!("DIR_OUTPUT = {}", path.display());
+    Box::leak(path.into_boxed_path())
+});
 
 /// Directory to store proofs on disc.
 const DIR_PROOFS: &str = "proofs";
@@ -74,7 +91,7 @@ pub trait ProverTester {
                 Self::DIR_ASSETS,
                 chrono::Utc::now().format("%Y%m%d_%H%M%S"),
             );
-            Path::new(DIR_OUTPUT).join(testrun)
+            DIR_OUTPUT.join(testrun)
         };
 
         // Set the path for the current test-run.
@@ -89,9 +106,10 @@ pub trait ProverTester {
     fn load_with_exe_fd(
         app_exe_fd: &str,
     ) -> eyre::Result<(PathBuf, AppConfig<SdkVmConfig>, PathBuf)> {
-        let path_app_config = Path::new(Self::PATH_PROJECT_ROOT).join(FD_APP_CONFIG);
+        let project_root = WORKSPACE_ROOT.join(Self::PATH_PROJECT_ROOT);
+        let path_app_config = project_root.join(FD_APP_CONFIG);
         let app_config = read_app_config(&path_app_config)?;
-        let path_assets = Path::new(Self::PATH_PROJECT_ROOT).join("openvm");
+        let path_assets = project_root.join("openvm");
         let path_app_exe = path_assets.join(app_exe_fd);
         Ok((path_app_config, app_config, path_app_exe))
     }
@@ -322,7 +340,8 @@ where
 
     // Dump verifier-only assets to disk.
     let (path_vm_config, path_root_committed_exe) = prover.dump_verifier(&path_assets)?;
-    let path_verifier_code = Path::new(T::PATH_PROJECT_ROOT)
+    let path_verifier_code = WORKSPACE_ROOT
+        .join(T::PATH_PROJECT_ROOT)
         .join("openvm")
         .join("verifier.bin");
     let verifier = scroll_zkvm_verifier::verifier::Verifier::setup(
@@ -346,3 +365,25 @@ where
         path_assets,
     ))
 }
+
+#[test]
+fn test_project_root() {
+    println!("Project root directory: {}", WORKSPACE_ROOT.display());
+    assert!(
+        WORKSPACE_ROOT.exists(),
+        "Project root directory does not exist"
+    );
+    let crates = WORKSPACE_ROOT.join("crates");
+    assert!(
+        crates.exists(),
+        "Expected 'crates' directory in project root"
+    );
+    assert!(
+        crates.join("circuits").exists(),
+        "Expected 'integration' directory in project root"
+    );
+    assert!(
+        crates.join("integration").exists(),
+        "Expected 'integration' directory in project root"
+    );
+}

crates/integration/src/testers/batch.rs

@@ -15,7 +15,7 @@ pub struct BatchProverTester;
 impl ProverTester for BatchProverTester {
     type Prover = BatchProverType;
 
-    const PATH_PROJECT_ROOT: &str = "./../circuits/batch-circuit";
+    const PATH_PROJECT_ROOT: &str = "crates/circuits/batch-circuit";
 
     const DIR_ASSETS: &str = "batch";
 
@@ -34,7 +34,7 @@ pub struct BatchTaskBuildingTester;
 impl ProverTester for BatchTaskBuildingTester {
     type Prover = BatchProverType;
 
-    const PATH_PROJECT_ROOT: &str = "./../circuits/batch-circuit";
+    const PATH_PROJECT_ROOT: &str = "crates/circuits/batch-circuit";
 
     const DIR_ASSETS: &str = "batch";
 

crates/integration/src/testers/bundle.rs

@@ -19,7 +19,7 @@ pub struct BundleProverTester;
 impl ProverTester for BundleProverTester {
     type Prover = BundleProverType;
 
-    const PATH_PROJECT_ROOT: &str = "./../circuits/bundle-circuit";
+    const PATH_PROJECT_ROOT: &str = "crates/circuits/bundle-circuit";
 
     const DIR_ASSETS: &str = "bundle";
 
@@ -49,7 +49,7 @@ pub struct BundleLocalTaskTester;
 impl ProverTester for BundleLocalTaskTester {
     type Prover = BundleProverType;
 
-    const PATH_PROJECT_ROOT: &str = "./../circuits/bundle-circuit";
+    const PATH_PROJECT_ROOT: &str = "crates/circuits/bundle-circuit";
 
     const DIR_ASSETS: &str = "bundle";
 

crates/integration/src/testers/chunk.rs

@@ -38,7 +38,7 @@ pub struct ChunkProverTester;
 impl ProverTester for ChunkProverTester {
     type Prover = ChunkProverType;
 
-    const PATH_PROJECT_ROOT: &str = "./../circuits/chunk-circuit";
+    const PATH_PROJECT_ROOT: &str = "crates/circuits/chunk-circuit";
 
     const DIR_ASSETS: &str = "chunk";
 
@@ -93,7 +93,7 @@ pub struct ChunkProverRv32Tester;
 impl ProverTester for ChunkProverRv32Tester {
     type Prover = ChunkProverTypeRv32;
 
-    const PATH_PROJECT_ROOT: &str = "./../circuits/chunk-circuit";
+    const PATH_PROJECT_ROOT: &str = "crates/circuits/chunk-circuit";
 
     const DIR_ASSETS: &str = "chunk";
 
@@ -163,7 +163,7 @@ pub struct MultiChunkProverTester;
 impl ProverTester for MultiChunkProverTester {
     type Prover = ChunkProverType;
 
-    const PATH_PROJECT_ROOT: &str = "./../circuits/chunk-circuit";
+    const PATH_PROJECT_ROOT: &str = "crates/circuits/chunk-circuit";
 
     const DIR_ASSETS: &str = "chunk";
 

crates/integration/src/utils/mod.rs

@@ -14,6 +14,7 @@ use scroll_zkvm_types::{
     batch::{BatchHeader, BatchHeaderV6, BatchHeaderV7},
     utils::keccak256,
 };
+use std::env;
 use vm_zstd::zstd_encode;
 
 fn is_l1_tx(tx: &Transaction) -> bool {
@@ -53,6 +54,31 @@ pub fn phase_base_directory() -> &'static str {
     }
 }
 
+pub fn get_rayon_threads() -> usize {
+    const MEMORY_PRESERVED_EACH_THREAD: u64 = 10 * 1024 * 1024 * 1024; // 10GB
+
+    if let Some(threads) = env::var("RAYON_NUM_THREADS")
+        .ok()
+        .and_then(|s| s.parse().ok())
+    {
+        eprintln!("RAYON_NUM_THREADS set, using {} threads", threads);
+        return threads;
+    }
+
+    let mut system = sysinfo::System::new();
+    system.refresh_cpu_list(sysinfo::CpuRefreshKind::nothing().with_frequency());
+    system.refresh_memory_specifics(sysinfo::MemoryRefreshKind::nothing().with_ram());
+    let free_memory = system.free_memory();
+    let max_threads = (free_memory / MEMORY_PRESERVED_EACH_THREAD) as usize;
+    let n_cpus = system.cpus().len();
+    let threads = std::cmp::min(max_threads, n_cpus);
+    eprintln!(
+        "RAYON_NUM_THREADS not set, using {} threads ({} CPUs, {} free memory)",
+        threads, n_cpus, free_memory
+    );
+    threads
+}
+
 #[derive(Debug)]
 pub struct LastHeader {
     pub batch_index: u64,