feat: backfill

rust/op-reth/crates/cli/src/commands/op_proofs/backfill.rs

@@ -0,0 +1,124 @@
+//! Command that backfills OP proofs storage to an older earliest block.
+
+use clap::Parser;
+use reth_cli::chainspec::ChainSpecParser;
+use reth_cli_commands::common::{AccessRights, CliNodeTypes, Environment, EnvironmentArgs};
+use reth_node_core::version::version_metadata;
+use reth_optimism_chainspec::OpChainSpec;
+use reth_optimism_node::args::ProofsStorageVersion;
+use reth_optimism_primitives::OpPrimitives;
+use reth_optimism_trie::{
+    BackfillJob, OpProofsProviderRO, OpProofsStore,
+    db::{MdbxProofsStorage, MdbxProofsStorageV2},
+};
+use reth_provider::{
+    BlockHashReader, BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory,
+    HeaderProvider, StageCheckpointReader, StorageChangeSetReader, StorageSettingsCache,
+};
+use std::{path::PathBuf, sync::Arc};
+use tracing::info;
+
+/// Backfills the proofs storage to an older earliest block.
+#[derive(Debug, Parser)]
+pub struct BackfillCommand<C: ChainSpecParser> {
+    #[command(flatten)]
+    env: EnvironmentArgs<C>,
+
+    /// The path to the storage DB for proofs history.
+    #[arg(
+        long = "proofs-history.storage-path",
+        value_name = "PROOFS_HISTORY_STORAGE_PATH",
+        required = true
+    )]
+    pub storage_path: PathBuf,
+
+    /// Target earliest block number after backfill.
+    #[arg(long = "proofs-history.target-earliest-block", value_name = "TARGET_EARLIEST_BLOCK")]
+    pub target_earliest_block: u64,
+
+    /// Storage schema version. Must match the version used when starting the node.
+    #[arg(
+        long = "proofs-history.storage-version",
+        value_name = "PROOFS_HISTORY_STORAGE_VERSION",
+        default_value = "v1"
+    )]
+    pub storage_version: ProofsStorageVersion,
+}
+
+impl<C: ChainSpecParser<ChainSpec = OpChainSpec>> BackfillCommand<C> {
+    /// Execute [`BackfillCommand`].
+    pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec, Primitives = OpPrimitives>>(
+        self,
+        runtime: reth_tasks::Runtime,
+    ) -> eyre::Result<()> {
+        info!(target: "reth::cli", "reth {} starting", version_metadata().short_version);
+        info!(target: "reth::cli", "Backfilling OP proofs storage at: {:?}", self.storage_path);
+
+        let Environment { provider_factory, .. } =
+            self.env.init::<N>(AccessRights::RO, runtime)?;
+
+        match self.storage_version {
+            ProofsStorageVersion::V1 => {
+                let storage: Arc<MdbxProofsStorage> = Arc::new(
+                    MdbxProofsStorage::new(&self.storage_path)
+                        .map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorage: {e}"))?,
+                );
+                Self::run_backfill(&provider_factory, storage, self.target_earliest_block)?;
+            }
+            ProofsStorageVersion::V2 => {
+                let storage: Arc<MdbxProofsStorageV2> = Arc::new(
+                    MdbxProofsStorageV2::new(&self.storage_path)
+                        .map_err(|e| eyre::eyre!("Failed to create MdbxProofsStorageV2: {e}"))?,
+                );
+                Self::run_backfill(&provider_factory, storage, self.target_earliest_block)?;
+            }
+        }
+
+        Ok(())
+    }
+
+    fn run_backfill<F, S>(
+        provider_factory: &F,
+        storage: S,
+        target_earliest_block: u64,
+    ) -> eyre::Result<()>
+    where
+        F: DatabaseProviderFactory,
+        F::Provider: DBProvider
+            + StageCheckpointReader
+            + ChangeSetReader
+            + StorageChangeSetReader
+            + BlockNumReader
+            + BlockHashReader
+            + HeaderProvider
+            + StorageSettingsCache
+            + Send,
+        S: OpProofsStore + Send,
+    {
+        let ro = storage.provider_ro()?;
+        let earliest = ro.get_earliest_block_number()?;
+        let latest = ro.get_latest_block_number()?;
+        drop(ro);
+        info!(
+            target: "reth::cli",
+            ?earliest,
+            ?latest,
+            target_earliest_block,
+            "Starting backfill job"
+        );
+
+        let provider = provider_factory
+            .database_provider_ro()
+            .map_err(|e| eyre::eyre!("Failed to open reth DB provider: {e}"))?;
+
+        BackfillJob::new(provider, storage).run(target_earliest_block)?;
+        Ok(())
+    }
+}
+
+impl<C: ChainSpecParser> BackfillCommand<C> {
+    /// Returns the underlying chain being used to run this command
+    pub const fn chain_spec(&self) -> Option<&Arc<C::ChainSpec>> {
+        Some(&self.env.chain)
+    }
+}

rust/op-reth/crates/cli/src/commands/op_proofs/mod.rs

@@ -8,6 +8,7 @@ use reth_optimism_primitives::OpPrimitives;
 use std::sync::Arc;
 
 pub mod init;
+pub mod backfill;
 pub mod prune;
 pub mod unwind;
 
@@ -26,6 +27,7 @@ impl<C: ChainSpecParser<ChainSpec = OpChainSpec>> Command<C> {
     ) -> eyre::Result<()> {
         match self.command {
             Subcommands::Init(cmd) => cmd.execute::<N>(runtime).await,
+            Subcommands::Backfill(cmd) => cmd.execute::<N>(runtime).await,
             Subcommands::Prune(cmd) => cmd.execute::<N>(runtime).await,
             Subcommands::Unwind(cmd) => cmd.execute::<N>(runtime).await,
         }
@@ -37,6 +39,7 @@ impl<C: ChainSpecParser> Command<C> {
     pub const fn chain_spec(&self) -> Option<&Arc<C::ChainSpec>> {
         match &self.command {
             Subcommands::Init(cmd) => cmd.chain_spec(),
+            Subcommands::Backfill(cmd) => cmd.chain_spec(),
             Subcommands::Prune(cmd) => cmd.chain_spec(),
             Subcommands::Unwind(cmd) => cmd.chain_spec(),
         }
@@ -49,6 +52,9 @@ pub enum Subcommands<C: ChainSpecParser> {
     /// Initialize the proofs storage with the current state of the chain
     #[command(name = "init")]
     Init(init::InitCommand<C>),
+    /// Backfill proofs history to an older earliest block
+    #[command(name = "backfill")]
+    Backfill(backfill::BackfillCommand<C>),
     /// Prune old proof history to reclaim space
     #[command(name = "prune")]
     Prune(prune::PruneCommand<C>),

rust/op-reth/crates/trie/Cargo.toml

@@ -21,6 +21,7 @@ reth-provider.workspace = true
 reth-revm.workspace = true
 reth-trie = { workspace = true, features = ["serde"] }
 reth-trie-common = { workspace = true, features = ["serde"] }
+reth-trie-db.workspace = true
 reth-codecs.workspace = true
 reth-tasks.workspace = true
 

rust/op-reth/crates/trie/src/api.rs

@@ -162,6 +162,126 @@ pub trait OpProofsProviderRw: OpProofsProviderRO {
     fn commit(self) -> OpProofsStorageResult<()>;
 }
 
+/// Provider for writing historical records for blocks older than the current window boundary.
+///
+/// Unlike [`OpProofsProviderRw::store_trie_updates`], which is strictly append-only (validates
+/// parent hash against `latest` and advances `latest`), this provider is designed for
+/// **prepend-style** writes that extend the window backward.  It does not touch the `latest`
+/// marker, and it does not enforce parent-hash ordering against `latest`.
+///
+/// The typical call sequence for one backfill step is:
+/// ```ignore
+/// let bp = storage.backfill_provider()?;
+/// bp.prepend_block(block_ref, diff)?;
+/// bp.commit()?;
+/// ```
+pub trait OpProofsBackfillProvider: OpProofsProviderRO {
+    /// Write historical changeset and history-bitmap entries for `block_ref`, and move the
+    /// `earliest` marker to `block_ref.parent`.
+    ///
+    /// `diff` contains:
+    /// - `sorted_trie_updates`: trie node **before-values** for `block_ref.block.number`
+    ///   (i.e. what each changed node looked like *before* the block executed).
+    /// - `sorted_post_state`: account / storage **before-values** for the same block.
+    ///
+    /// The implementation must **not** update the `latest` marker and must **not**
+    /// validate `diff` against the current `latest` block.
+    fn prepend_block(
+        &self,
+        block_ref: BlockWithParent,
+        diff: BlockStateDiff,
+    ) -> OpProofsStorageResult<WriteCounts>;
+
+    /// Commit the transaction. Consumes the provider.
+    fn commit(self) -> OpProofsStorageResult<()>;
+}
+
+/// Blanket impl of [`OpProofsProviderRO`] for shared references.
+///
+/// This allows passing `&bp` (where `bp: OpProofsBackfillProvider + OpProofsProviderRO`)
+/// to APIs that require `P: OpProofsProviderRO + Clone`. Since `&T: Copy`, cloning a
+/// reference is free, enabling `StateRoot::overlay_root(&bp, ...)` to work without
+/// requiring the underlying provider to implement `Clone`.
+impl<'a, T: OpProofsProviderRO + 'a> OpProofsProviderRO for &'a T {
+    type StorageTrieCursor<'tx>
+        = T::StorageTrieCursor<'tx>
+    where
+        Self: 'tx,
+        T: 'tx;
+    type AccountTrieCursor<'tx>
+        = T::AccountTrieCursor<'tx>
+    where
+        Self: 'tx,
+        T: 'tx;
+    type StorageCursor<'tx>
+        = T::StorageCursor<'tx>
+    where
+        Self: 'tx,
+        T: 'tx;
+    type AccountHashedCursor<'tx>
+        = T::AccountHashedCursor<'tx>
+    where
+        Self: 'tx,
+        T: 'tx;
+
+    fn get_earliest_block_number(&self) -> crate::OpProofsStorageResult<Option<(u64, B256)>> {
+        T::get_earliest_block_number(self)
+    }
+
+    fn get_latest_block_number(&self) -> crate::OpProofsStorageResult<Option<(u64, B256)>> {
+        T::get_latest_block_number(self)
+    }
+
+    fn storage_trie_cursor<'tx>(
+        &self,
+        hashed_address: B256,
+        max_block_number: u64,
+    ) -> crate::OpProofsStorageResult<Self::StorageTrieCursor<'tx>>
+    where
+        'a: 'tx,
+    {
+        T::storage_trie_cursor(self, hashed_address, max_block_number)
+    }
+
+    fn account_trie_cursor<'tx>(
+        &self,
+        max_block_number: u64,
+    ) -> crate::OpProofsStorageResult<Self::AccountTrieCursor<'tx>>
+    where
+        'a: 'tx,
+    {
+        T::account_trie_cursor(self, max_block_number)
+    }
+
+    fn storage_hashed_cursor<'tx>(
+        &self,
+        hashed_address: B256,
+        max_block_number: u64,
+    ) -> crate::OpProofsStorageResult<Self::StorageCursor<'tx>>
+    where
+        'a: 'tx,
+    {
+        T::storage_hashed_cursor(self, hashed_address, max_block_number)
+    }
+
+    fn account_hashed_cursor<'tx>(
+        &self,
+        max_block_number: u64,
+    ) -> crate::OpProofsStorageResult<Self::AccountHashedCursor<'tx>>
+    where
+        'a: 'tx,
+    {
+        T::account_hashed_cursor(self, max_block_number)
+    }
+
+    fn fetch_trie_updates(
+        &self,
+        block_number: u64,
+    ) -> crate::OpProofsStorageResult<BlockStateDiff> {
+        T::fetch_trie_updates(self, block_number)
+    }
+}
+
 /// Factory trait for creating providers to interact with the proofs storage.
 #[auto_impl(Arc)]
 pub trait OpProofsStore: Send + Sync + Debug {
@@ -180,6 +300,11 @@ pub trait OpProofsStore: Send + Sync + Debug {
     where
         Self: 'a;
 
+    /// The backfill provider type created by the factory.
+    type BackfillProvider<'a>: OpProofsBackfillProvider + 'a
+    where
+        Self: 'a;
+
     /// Create a read-only provider for interacting with the proofs storage.
     fn provider_ro<'a>(&'a self) -> OpProofsStorageResult<Self::ProviderRO<'a>>;
 
@@ -188,6 +313,9 @@ pub trait OpProofsStore: Send + Sync + Debug {
 
     /// Create an initialization provider for interacting with the proofs storage.
     fn initialization_provider<'a>(&'a self) -> OpProofsStorageResult<Self::Initializer<'a>>;
+
+    /// Create a backfill provider for prepend-style writes that extend the window backward.
+    fn backfill_provider<'a>(&'a self) -> OpProofsStorageResult<Self::BackfillProvider<'a>>;
 }
 
 /// Status of the initial state anchor.