Various changes related to the fork detection script

Cargo.toml

@@ -149,6 +149,7 @@ cfg-if = "1.0"
 chacha20poly1305 = "0.10"
 chrono = "0.4"
 clap = "4.5"
+csv = "1.3"
 ctor = "0.2"
 criterion = "0.5"
 crossterm = "0.28"

blockprod/src/lib.rs

@@ -274,6 +274,7 @@ mod tests {
             max_db_commit_attempts: Default::default(),
             max_orphan_blocks: Default::default(),
             min_max_bootstrap_import_buffer_sizes: Default::default(),
+            allow_checkpoints_mismatch: Default::default(),
         };
 
         let mempool_config = MempoolConfig::new();

build-tools/docker/build.py

@@ -1,7 +1,12 @@
-import toml
+import argparse
 import os
+import pathlib
 import subprocess
-import argparse
+import toml
+
+
+ROOT_DIR = pathlib.Path(__file__).resolve().parent.parent.parent
+ROOT_CARGO_TOML = ROOT_DIR.joinpath("Cargo.toml")
 
 
 def get_cargo_version(cargo_toml_path):
@@ -44,7 +49,7 @@ def build_docker_image(dockerfile_path, image_name, tags, num_jobs=None):
 
     try:
         # Run the command
-        subprocess.check_call(command, shell=True)
+        subprocess.check_call(command, shell=True, cwd=ROOT_DIR)
         print(f"Built {image_name} successfully (the tags are: {full_tags}).")
     except subprocess.CalledProcessError as error:
         print(f"Failed to build {image_name}: {error}")
@@ -121,7 +126,7 @@ def main():
     parser.add_argument('--local_tags', nargs='*', help='Additional tags to apply (these won\'t be pushed)', default=[])
     args = parser.parse_args()
 
-    version = args.version if args.version else get_cargo_version("Cargo.toml")
+    version = args.version if args.version else get_cargo_version(ROOT_CARGO_TOML)
     # Note: the CI currently takes the version from the release tag, so it always starts with "v",
     # but the version from Cargo.toml doesn't have this prefix.
     version = version.removeprefix("v")

build-tools/docker/example-mainnet/docker-compose.yml

@@ -65,6 +65,7 @@ services:
       - node-daemon
     environment:
       <<: *ml-common-env
+      ML_API_SCANNER_DAEMON_NETWORK: mainnet
       ML_API_SCANNER_DAEMON_POSTGRES_HOST: api-postgres-db
       ML_API_SCANNER_DAEMON_POSTGRES_USER: $API_SERVER_POSTGRES_USER
       ML_API_SCANNER_DAEMON_POSTGRES_PASSWORD: $API_SERVER_POSTGRES_PASSWORD
@@ -83,6 +84,7 @@ services:
       - node-daemon
     environment:
       <<: *ml-common-env
+      ML_API_WEB_SRV_NETWORK: mainnet
       ML_API_WEB_SRV_BIND_ADDRESS: 0.0.0.0:3000
       ML_API_WEB_SRV_POSTGRES_HOST: api-postgres-db
       ML_API_WEB_SRV_POSTGRES_USER: $API_SERVER_POSTGRES_USER

chainstate/src/config.rs

@@ -38,17 +38,24 @@ make_config_setting!(MaxTipAge, Duration, Duration::from_secs(60 * 60 * 24));
 pub struct ChainstateConfig {
     /// The number of maximum attempts to process a block.
     pub max_db_commit_attempts: MaxDbCommitAttempts,
+
     /// The maximum capacity of the orphan blocks pool.
     pub max_orphan_blocks: MaxOrphanBlocks,
+
     /// When importing bootstrap file, this controls the buffer sizes (min, max)
     /// (see bootstrap import function for more information)
     pub min_max_bootstrap_import_buffer_sizes: MinMaxBootstrapImportBufferSizes,
+
     /// The initial block download is finished if the difference between the current time and the
     /// tip time is less than this value.
     pub max_tip_age: MaxTipAge,
+
     /// If true, additional computationally-expensive consistency checks will be performed by
     /// the chainstate. The default value depends on the chain type.
     pub enable_heavy_checks: Option<bool>,
+
+    /// If true, blocks and block headers will not be rejected if checkpoints mismatch is detected.
+    pub allow_checkpoints_mismatch: Option<bool>,
 }
 
 impl ChainstateConfig {
@@ -90,4 +97,8 @@ impl ChainstateConfig {
             ChainType::Regtest => true,
         }
     }
+
+    pub fn checkpoints_mismatch_allowed(&self) -> bool {
+        self.allow_checkpoints_mismatch.unwrap_or(false)
+    }
 }

chainstate/src/detail/ban_score.rs

@@ -317,13 +317,13 @@ impl BanScore for CheckBlockError {
             CheckBlockError::MerkleRootCalculationFailed(_, _) => 100,
             CheckBlockError::BlockRewardMaturityError(err) => err.ban_score(),
             CheckBlockError::PropertyQueryError(_) => 100,
-            CheckBlockError::CheckpointMismatch(_, _) => 100,
-            CheckBlockError::ParentCheckpointMismatch(_, _, _) => 100,
+            CheckBlockError::CheckpointMismatch { .. } => 100,
             CheckBlockError::GetAncestorError(_) => 100,
-            CheckBlockError::AttemptedToAddBlockBeforeReorgLimit(_, _, _) => 100,
+            CheckBlockError::AttemptedToAddBlockBeforeReorgLimit { .. } => 100,
             CheckBlockError::EpochSealError(err) => err.ban_score(),
             CheckBlockError::InvalidParent { .. } => 100,
             CheckBlockError::InMemoryReorgFailed(err) => err.ban_score(),
+            CheckBlockError::InvalidBlockAlreadyProcessed(_) => 100,
         }
     }
 }

chainstate/src/detail/chainstateref/mod.rs

@@ -76,7 +76,7 @@ pub use in_memory_reorg::InMemoryReorgError;
 
 pub struct ChainstateRef<'a, S, V> {
     chain_config: &'a ChainConfig,
-    _chainstate_config: &'a ChainstateConfig,
+    chainstate_config: &'a ChainstateConfig,
     tx_verification_strategy: &'a V,
     db_tx: S,
     time_getter: &'a TimeGetter,
@@ -141,7 +141,7 @@ impl<'a, S: BlockchainStorageRead, V: TransactionVerificationStrategy> Chainstat
     ) -> Self {
         ChainstateRef {
             chain_config,
-            _chainstate_config: chainstate_config,
+            chainstate_config,
             db_tx,
             tx_verification_strategy,
             time_getter,
@@ -157,7 +157,7 @@ impl<'a, S: BlockchainStorageRead, V: TransactionVerificationStrategy> Chainstat
     ) -> Self {
         ChainstateRef {
             chain_config,
-            _chainstate_config: chainstate_config,
+            chainstate_config,
             db_tx,
             tx_verification_strategy,
             time_getter,
@@ -457,22 +457,45 @@ impl<'a, S: BlockchainStorageRead, V: TransactionVerificationStrategy> Chainstat
         Ok(result)
     }
 
+    fn enforce_checkpoint_impl(
+        &self,
+        height: BlockHeight,
+        expected: &Id<GenBlock>,
+        given: &Id<GenBlock>,
+    ) -> Result<(), CheckBlockError> {
+        if given != expected {
+            // Note: we only log the mismatch if we're going to ignore it (because if it's
+            // not ignored, we'll log the error anyway).
+            if self.chainstate_config.checkpoints_mismatch_allowed() {
+                log::warn!(
+                    "Checkpoint mismatch at height {}, expected: {:x}, actual: {:x}",
+                    height,
+                    expected,
+                    given,
+                );
+            } else {
+                return Err(CheckBlockError::CheckpointMismatch {
+                    height,
+                    expected: *expected,
+                    given: *given,
+                });
+            }
+        }
+
+        Ok(())
+    }
+
     // If the header height is at an exact checkpoint height, check that the block id matches the checkpoint id.
     // Return true if the header height is at an exact checkpoint height.
     fn enforce_exact_checkpoint_assuming_height(
         &self,
         header: &SignedBlockHeader,
         header_height: BlockHeight,
     ) -> Result<bool, CheckBlockError> {
-        if let Some(e) = self.chain_config.height_checkpoints().checkpoint_at_height(&header_height)
+        if let Some(expected_id) =
+            self.chain_config.height_checkpoints().checkpoint_at_height(&header_height)
         {
-            let expected_id = Id::<Block>::new(e.to_hash());
-            if expected_id != header.get_id() {
-                return Err(CheckBlockError::CheckpointMismatch(
-                    expected_id,
-                    header.get_id(),
-                ));
-            }
+            self.enforce_checkpoint_impl(header_height, expected_id, &header.get_id().into())?;
             Ok(true)
         } else {
             Ok(false)
@@ -499,17 +522,13 @@ impl<'a, S: BlockchainStorageRead, V: TransactionVerificationStrategy> Chainstat
 
         let parent_checkpoint_block_index =
             self.get_ancestor(&prev_block_index, expected_checkpoint_height)?;
-
         let parent_checkpoint_id = parent_checkpoint_block_index.block_id();
 
-        if parent_checkpoint_id != expected_checkpoint_id {
-            return Err(CheckBlockError::ParentCheckpointMismatch(
-                expected_checkpoint_height,
-                expected_checkpoint_id,
-                parent_checkpoint_id,
-            ));
-        }
-
+        self.enforce_checkpoint_impl(
+            expected_checkpoint_height,
+            &expected_checkpoint_id,
+            &parent_checkpoint_id,
+        )?;
         Ok(())
     }
 
@@ -564,11 +583,11 @@ impl<'a, S: BlockchainStorageRead, V: TransactionVerificationStrategy> Chainstat
         if common_ancestor_height < min_allowed_height {
             let tip_block_height = self.get_best_block_index()?.block_height();
 
-            return Err(CheckBlockError::AttemptedToAddBlockBeforeReorgLimit(
+            return Err(CheckBlockError::AttemptedToAddBlockBeforeReorgLimit {
                 common_ancestor_height,
                 tip_block_height,
                 min_allowed_height,
-            ));
+            });
         }
 
         Ok(())
@@ -599,8 +618,45 @@ impl<'a, S: BlockchainStorageRead, V: TransactionVerificationStrategy> Chainstat
         Ok(parent_block_index)
     }
 
+    /// This function is intended to be used in check_block and check_block_header.
+    ///
+    /// Return true if the block already exists in the chainstate and has an "ok" status
+    /// with the validation stage CheckBlockOk or later.
+    /// If it has a non-"ok" status, return an error.
+    /// If the block is new, or if its validation stage is below CheckBlockOk (i.e. it's Unchecked),
+    /// return false.
+    fn skip_check_block_because_block_exists_and_is_checked(
+        &self,
+        block_id: &Id<Block>,
+    ) -> Result<bool, CheckBlockError> {
+        if let Some(block_index) = self.get_block_index(block_id)? {
+            let status = block_index.status();
+
+            if status.is_ok() {
+                let checked = status.last_valid_stage() >= BlockValidationStage::CheckBlockOk;
+                Ok(checked)
+            } else {
+                Err(CheckBlockError::InvalidBlockAlreadyProcessed(*block_id))
+            }
+        } else {
+            Ok(false)
+        }
+    }
+
     #[log_error]
     pub fn check_block_header(&self, header: &SignedBlockHeader) -> Result<(), CheckBlockError> {
+        let header = WithId::new(header);
+        if self.skip_check_block_because_block_exists_and_is_checked(&WithId::id(&header))? {
+            return Ok(());
+        }
+
+        self.check_block_header_impl(&header)
+    }
+
+    fn check_block_header_impl(
+        &self,
+        header: &WithId<&SignedBlockHeader>,
+    ) -> Result<(), CheckBlockError> {
         let parent_block_index = self.check_block_parent(header)?;
         self.check_header_size(header)?;
         self.enforce_checkpoints(header)?;
@@ -662,7 +718,7 @@ impl<'a, S: BlockchainStorageRead, V: TransactionVerificationStrategy> Chainstat
         ensure!(
             block_timestamp.as_duration_since_epoch() <= current_time_as_secs + max_future_offset,
             CheckBlockError::BlockFromTheFuture {
-                block_id: header.block_id(),
+                block_id: WithId::id(header),
                 block_timestamp,
                 current_time
             },
@@ -833,7 +889,14 @@ impl<'a, S: BlockchainStorageRead, V: TransactionVerificationStrategy> Chainstat
 
     #[log_error]
     pub fn check_block(&self, block: &WithId<Block>) -> Result<(), CheckBlockError> {
-        self.check_block_header(block.header())?;
+        let header_with_id = WithId::as_sub_obj(block);
+        if self
+            .skip_check_block_because_block_exists_and_is_checked(&WithId::id(&header_with_id))?
+        {
+            return Ok(());
+        }
+
+        self.check_block_header_impl(&header_with_id)?;
 
         self.check_block_size(block).map_err(CheckBlockError::BlockSizeError)?;
 

chainstate/src/detail/error.rs

@@ -159,14 +159,25 @@ pub enum CheckBlockError {
     InvalidBlockRewardOutputType(Id<Block>),
     #[error("Block reward maturity error: {0}")]
     BlockRewardMaturityError(#[from] tx_verifier::timelock_check::OutputMaturityError),
-    #[error("Checkpoint mismatch: expected {0} vs given {1}")]
-    CheckpointMismatch(Id<Block>, Id<Block>),
-    #[error("Parent checkpoint mismatch at height {0}: expected {1} vs given {2}")]
-    ParentCheckpointMismatch(BlockHeight, Id<GenBlock>, Id<GenBlock>),
+    #[error("Checkpoint mismatch at height {height}: expected {expected:x}, given {given:x}")]
+    CheckpointMismatch {
+        height: BlockHeight,
+        expected: Id<GenBlock>,
+        given: Id<GenBlock>,
+    },
     #[error("CRITICAL: Failed to retrieve ancestor of submitted block: {0}")]
     GetAncestorError(#[from] GetAncestorError),
-    #[error("Attempted to add a block before reorg limit (attempted at height: {0} while current height is: {1} and min allowed is: {2})")]
-    AttemptedToAddBlockBeforeReorgLimit(BlockHeight, BlockHeight, BlockHeight),
+    #[error(
+        "Attempted to add a block before reorg limit (attempted at height: {} while current height is: {} and min allowed is: {})",
+        common_ancestor_height,
+        tip_block_height,
+        min_allowed_height
+    )]
+    AttemptedToAddBlockBeforeReorgLimit {
+        common_ancestor_height: BlockHeight,
+        tip_block_height: BlockHeight,
+        min_allowed_height: BlockHeight,
+    },
     #[error("TransactionVerifier error: {0}")]
     TransactionVerifierError(#[from] TransactionVerifierStorageError),
     #[error("Error during sealing an epoch: {0}")]
@@ -178,6 +189,8 @@ pub enum CheckBlockError {
     },
     #[error("In-memory reorg failed: {0}")]
     InMemoryReorgFailed(#[from] InMemoryReorgError),
+    #[error("Block {0} has already been processed and marked as invalid")]
+    InvalidBlockAlreadyProcessed(Id<Block>),
 }
 
 #[derive(Error, Debug, PartialEq, Eq, Clone)]

chainstate/src/detail/error_classification.rs

@@ -169,10 +169,12 @@ impl BlockProcessingErrorClassification for CheckBlockError {
             | CheckBlockError::ParentBlockMissing { .. }
             | CheckBlockError::BlockTimeOrderInvalid(_, _)
             | CheckBlockError::InvalidBlockRewardOutputType(_)
-            | CheckBlockError::CheckpointMismatch(_, _)
-            | CheckBlockError::ParentCheckpointMismatch(_, _, _)
-            | CheckBlockError::AttemptedToAddBlockBeforeReorgLimit(_, _, _)
-            | CheckBlockError::InvalidParent { .. } => BlockProcessingErrorClass::BadBlock,
+            | CheckBlockError::CheckpointMismatch { .. }
+            | CheckBlockError::AttemptedToAddBlockBeforeReorgLimit { .. }
+            | CheckBlockError::InvalidParent { .. }
+            | CheckBlockError::InvalidBlockAlreadyProcessed(_) => {
+                BlockProcessingErrorClass::BadBlock
+            }
 
             CheckBlockError::BlockFromTheFuture { .. } => {
                 BlockProcessingErrorClass::TemporarilyBadBlock