Add context gap aware finalization (#995)

* Add context gap aware finalization

* Fix build

Author: Warchant

cmake/functions.cmake

@@ -46,9 +46,13 @@ function(addtest test_name)
             gtest
             Threads::Threads
             )
+
+    # generate 7-digit random number to run UT with different seeds
+    string(RANDOM LENGTH 7 ALPHABET 0123456789 _SEED)
+    math(EXPR number "${_SEED} + 0")  # Remove extra leading 0s.
     add_test(
             NAME ${test_name}
-            COMMAND $<TARGET_FILE:${test_name}>
+            COMMAND $<TARGET_FILE:${test_name}> ${_SEED}
     )
     set_target_properties(${test_name} PROPERTIES
             CXX_STANDARD 17

include/veriblock/pop/algorithm.hpp

@@ -146,6 +146,14 @@ T& as_mut(const T& t) {
   return const_cast<T&>(t);
 }
 
+//! returns min value in a container (vector, array...), or "default" value if
+//! container is empty
+template <typename T, typename Container>
+const T min_or_default(Container& c, const T default_) {
+  const auto it = std::min_element(c.begin(), c.end());
+  return it == c.end() ? default_ : *it;
+}
+
 }  // namespace altintegration
 
 #endif  // VERIBLOCK_POP_CPP_ALGORITHM_HPP

include/veriblock/pop/blockchain/base_block_tree.hpp

@@ -6,6 +6,7 @@
 #ifndef ALTINTEGRATION_BASE_BLOCK_TREE_HPP
 #define ALTINTEGRATION_BASE_BLOCK_TREE_HPP
 
+#include <limits>
 #include <unordered_map>
 #include <unordered_set>
 #include <veriblock/pop/algorithm.hpp>
@@ -431,8 +432,12 @@ struct BaseBlockTree {
 
  protected:
   //! @private
-  void finalizeBlocks(int32_t maxReorgBlocks,
-                      int32_t preserveBlocksBehindFinal) {
+  void finalizeBlocks(
+      int32_t maxReorgBlocks,
+      int32_t preserveBlocksBehindFinal,
+      // we should not finalize blocks
+      // above this height
+      int32_t maxFinalizeBlockHeight = std::numeric_limits<int32_t>::max()) {
     auto* tip = this->getBestChain().tip();
     VBK_ASSERT_MSG(tip, "%s tree must be bootstrapped", block_t::name());
     VBK_ASSERT(!this->isLoadingBlocks_);
@@ -447,6 +452,17 @@ struct BaseBlockTree {
     firstBlockHeight = std::max(bootstrapBlockHeight, firstBlockHeight);
     auto* finalizedIndex = this->getBestChain()[firstBlockHeight];
     VBK_ASSERT(finalizedIndex != nullptr);
+    if (finalizedIndex->getHeight() >= maxFinalizeBlockHeight) {
+      // we should not finalize blocks with height higher than
+      // maxFinalizeBlockHeight
+      VBK_LOG_INFO(
+          "Skipping finalization of %s because its height >= %d (max "
+          "possible height to finalize)",
+          finalizedIndex->toShortPrettyString(),
+          maxFinalizeBlockHeight);
+      return;
+    }
+
     this->finalizeBlockImpl(*finalizedIndex, preserveBlocksBehindFinal);
   }
 
@@ -967,13 +983,17 @@ struct BaseBlockTree {
     // update active chain
     VBK_ASSERT(firstBlockHeight >= rootBlockHeight);
     size_t deallocatedBlocks = firstBlockHeight - rootBlockHeight;
-    if (deallocatedBlocks > 0) {
-      VBK_LOG_WARN(
-          "%s tree deallocated blocks: %d", block_t::name(), deallocatedBlocks);
-    }
     activeChain_ = Chain<index_t>(firstBlockHeight, activeChain_.tip());
     appliedBlockCount = activeChain_.blocksCount();
 
+    if (deallocatedBlocks > 0) {
+      VBK_LOG_WARN("Deallocated %d blocks in %s tree. Active chain %d..%d",
+                   deallocatedBlocks,
+                   block_t::name(),
+                   activeChain_.first()->getHeight(),
+                   activeChain_.tip()->getHeight());
+    }
+
     // fourth, mark `index` and all predecessors as finalized
     index_t* ptr = finalizedBlock;
     while (ptr != nullptr && !ptr->finalized) {

include/veriblock/pop/blockchain/blocktree.hpp

@@ -9,16 +9,16 @@
 #include <algorithm>
 #include <memory>
 #include <unordered_map>
-#include <veriblock/pop/fmt.hpp>
-#include <veriblock/pop/stateless_validation.hpp>
-#include <veriblock/pop/storage/block_reader.hpp>
-#include <veriblock/pop/validation_state.hpp>
 
 #include "base_block_tree.hpp"
 #include "block_index.hpp"
 #include "blockchain_util.hpp"
 #include "chain.hpp"
 #include "tree_algo.hpp"
+#include "veriblock/pop/fmt.hpp"
+#include "veriblock/pop/stateless_validation.hpp"
+#include "veriblock/pop/storage/block_reader.hpp"
+#include "veriblock/pop/validation_state.hpp"
 
 namespace altintegration {
 

src/pop/blockchain/pop/vbk_block_tree.cpp

@@ -7,6 +7,7 @@
 
 #include <algorithm>
 #include <cstddef>
+#include <limits>
 #include <map>
 #include <memory>
 #include <set>
@@ -459,9 +460,15 @@ void VbkBlockTree::finalizeBlocks() {
   VBK_ASSERT(!this->isLoadingBlocks_);
   VBK_ASSERT(appliedBlockCount == activeChain_.blocksCount());
 
+  const auto* btctip = btc().getBestChain().tip();
+  VBK_ASSERT(btctip != nullptr);
+  int32_t minVbkRefHeight =
+      min_or_default(btctip->getRefs(), std::numeric_limits<int32_t>::max());
+
   // first, finalize VBK
   base::finalizeBlocks(this->getParams().getMaxReorgBlocks(),
-                       this->getParams().preserveBlocksBehindFinal());
+                       this->getParams().preserveBlocksBehindFinal(),
+                       /*maxFinalizeBlockHeihht=*/minVbkRefHeight);
 
   // then, finalize BTC
   btc().finalizeBlocks();

test/pop/blockchain/altblock_finalization_test_2.cpp

@@ -22,6 +22,13 @@ struct AltBlockFinalization2 : public MemPoolFixture {
     tip = mineAltBlocks(alttree.getRoot(), 100);
     ASSERT_TRUE(alttree.setState(*tip, state));
     totalBlocks = alttree.getBlocks().size();
+
+    alttree.btc().getBestChain().tip()->addRef(1000000u);
+  }
+
+  void disableBtcContextGapCheck() {
+    alttree.btc().getBestChain().tip()->removeRef(0);
+    alttree.btc().getBestChain().tip()->addRef(1000000u);
   }
 };
 
@@ -61,6 +68,8 @@ TEST_F(AltBlockFinalization2, FinalizeVbkTip) {
   // save state
   save(alttree);
 
+  // this allows to bypass context gap finalization check
+  disableBtcContextGapCheck();
   alttree.finalizeBlocks();
 
   ASSERT_EQ(alttree.getBlocks().size(), 2);
@@ -106,6 +115,7 @@ TEST_F(AltBlockFinalization2, FinalizeMaxVbks) {
   save(alttree);
 
   // finalize block
+  disableBtcContextGapCheck();
   alttree.finalizeBlocks();
 
   ASSERT_EQ(alttree.getBlocks().size(), 2);
@@ -160,6 +170,7 @@ TEST_F(AltBlockFinalization2, FinalizedVbkBlock_1) {
   auto *vbktip = alttree.vbk().getBestChain().tip();
 
   // finalize block
+  disableBtcContextGapCheck();
   alttree.finalizeBlocks();
 
   // check the state after finalization
@@ -221,6 +232,7 @@ TEST_F(AltBlockFinalization2, FinalizedVbkBlock_2) {
   auto *vbktip = alttree.vbk().getBestChain().tip();
 
   // finalize block
+  disableBtcContextGapCheck();
   alttree.finalizeBlocks();
 
   // check the state after finalization