add basic snapshot test, modify block log to support partial logs, fix several bugs that result from that concept

Author: b1bart

libraries/chain/authorization_manager.cpp

@@ -56,10 +56,10 @@ namespace eosio { namespace chain {
    void authorization_manager::read_from_snapshot( const snapshot_reader_ptr& snapshot ) {
       authorization_index_set::walk_indices([this, &snapshot]( auto utils ){
          snapshot->read_section<typename decltype(utils)::index_t::value_type>([this]( auto& section ) {
-            bool done = section.empty();
-            while(!done) {
-               decltype(utils)::create(_db, [&section]( auto &row ) {
-                  section.read_row(row);
+            bool more = !section.empty();
+            while(more) {
+               decltype(utils)::create(_db, [&section, &more]( auto &row ) {
+                  more = section.read_row(row);
                });
             }
          });

libraries/chain/block_log.cpp

@@ -12,7 +12,15 @@
 
 namespace eosio { namespace chain {
 
-   const uint32_t block_log::supported_version = 1;
+   const uint32_t block_log::min_supported_version = 1;
+
+   /**
+    * History:
+    * Version 1: complete block log from genesis
+    * Version 2: adds optional partial block log, cannot be used for replay without snapshot
+    *            this is in the form of an first_block_num that is written immediately after the version
+    */
+   const uint32_t block_log::max_supported_version = 2;
 
    namespace detail {
       class block_log_impl {
@@ -26,6 +34,8 @@ namespace eosio { namespace chain {
             bool                     block_write;
             bool                     index_write;
             bool                     genesis_written_to_block_log = false;
+            uint32_t                 version = 0;
+            uint32_t                 first_block_num = 0;
 
             inline void check_block_read() {
                if (block_write) {
@@ -124,14 +134,21 @@ namespace eosio { namespace chain {
          ilog("Log is nonempty");
          my->check_block_read();
          my->block_stream.seekg( 0 );
-         uint32_t version = 0;
-         my->block_stream.read( (char*)&version, sizeof(version) );
-         EOS_ASSERT( version > 0, block_log_exception, "Block log was not setup properly with genesis information." );
-         EOS_ASSERT( version == block_log::supported_version, block_log_unsupported_version,
-                    "Unsupported version of block log. Block log version is ${version} while code supports version ${supported}",
-                    ("version", version)("supported", block_log::supported_version) );
+         my->version = 0;
+         my->block_stream.read( (char*)&my->version, sizeof(my->version) );
+         EOS_ASSERT( my->version > 0, block_log_exception, "Block log was not setup properly" );
+         EOS_ASSERT( my->version >= min_supported_version && my->version <= max_supported_version, block_log_unsupported_version,
+                 "Unsupported version of block log. Block log version is ${version} while code supports version(s) [${min},${max}]",
+                 ("version", my->version)("min", block_log::min_supported_version)("max", block_log::max_supported_version) );
+
 
          my->genesis_written_to_block_log = true; // Assume it was constructed properly.
+         if (my->version > 1){
+            my->first_block_num = 0;
+            my->block_stream.read( (char*)&my->first_block_num, sizeof(my->first_block_num) );
+            EOS_ASSERT(my->first_block_num > 0, block_log_exception, "Block log is malformed, first recorded block number is 0 but must be greater than or equal to 1");
+         }
+
          my->head = read_head();
          my->head_id = my->head->id();
 
@@ -176,11 +193,11 @@ namespace eosio { namespace chain {
          my->check_index_write();
 
          uint64_t pos = my->block_stream.tellp();
-         EOS_ASSERT(my->index_stream.tellp() == sizeof(uint64_t) * (b->block_num() - 1),
+         EOS_ASSERT(my->index_stream.tellp() == sizeof(uint64_t) * (b->block_num() - my->first_block_num),
                    block_log_append_fail,
                    "Append to index file occuring at wrong position.",
                    ("position", (uint64_t) my->index_stream.tellp())
-                   ("expected", (b->block_num() - 1) * sizeof(uint64_t)));
+                   ("expected", (b->block_num() - my->first_block_num) * sizeof(uint64_t)));
          auto data = fc::raw::pack(*b);
          my->block_stream.write(data.data(), data.size());
          my->block_stream.write((char*)&pos, sizeof(pos));
@@ -200,44 +217,50 @@ namespace eosio { namespace chain {
       my->index_stream.flush();
    }
 
-   uint64_t block_log::reset_to_genesis( const genesis_state& gs, const signed_block_ptr& genesis_block ) {
-      if( my->block_stream.is_open() )
+   void block_log::reset( const genesis_state& gs, const signed_block_ptr& first_block, uint32_t first_block_num ) {
+      if (my->block_stream.is_open())
          my->block_stream.close();
-      if( my->index_stream.is_open() )
+      if (my->index_stream.is_open())
          my->index_stream.close();
 
-      fc::remove_all( my->block_file );
-      fc::remove_all( my->index_file );
+      fc::remove_all(my->block_file);
+      fc::remove_all(my->index_file);
 
       my->block_stream.open(my->block_file.generic_string().c_str(), LOG_WRITE);
       my->index_stream.open(my->index_file.generic_string().c_str(), LOG_WRITE);
       my->block_write = true;
       my->index_write = true;
 
-      auto data = fc::raw::pack( gs );
-      uint32_t version = 0; // version of 0 is invalid; it indicates that the genesis was not properly written to the block log
-      my->block_stream.write( (char*)&version, sizeof(version) );
-      my->block_stream.write( data.data(), data.size() );
+      auto data = fc::raw::pack(gs);
+      my->version = 0; // version of 0 is invalid; it indicates that the genesis was not properly written to the block log
+      my->first_block_num = first_block_num;
+      my->block_stream.write((char*)&my->version, sizeof(my->version));
+      my->block_stream.write((char*)&my->first_block_num, sizeof(my->first_block_num));
+      my->block_stream.write(data.data(), data.size());
       my->genesis_written_to_block_log = true;
 
-      auto ret = append( genesis_block );
+      // append a totem to indicate the division between blocks and header
+      auto totem = npos;
+      my->block_stream.write((char*)&totem, sizeof(totem));
+
+      if (first_block) {
+         auto ret = append(first_block);
+      }
 
       auto pos = my->block_stream.tellp();
 
       my->block_stream.close();
       my->block_stream.open(my->block_file.generic_string().c_str(), std::ios::in | std::ios::out | std::ios::binary ); // Bypass append-only writing just once
 
-      static_assert( block_log::supported_version > 0, "a version number of zero is not supported" );
-      version = block_log::supported_version;
+      static_assert( block_log::max_supported_version > 0, "a version number of zero is not supported" );
+      my->version = block_log::max_supported_version;
       my->block_stream.seekp( 0 );
-      my->block_stream.write( (char*)&version, sizeof(version) ); // Finally write actual version to disk.
+      my->block_stream.write( (char*)&my->version, sizeof(my->version) );
       my->block_stream.seekp( pos );
       flush();
 
       my->block_write = false;
       my->check_block_write(); // Reset to append-only writing.
-
-      return ret;
    }
 
    std::pair<signed_block_ptr, uint64_t> block_log::read_block(uint64_t pos)const {
@@ -266,10 +289,9 @@ namespace eosio { namespace chain {
 
    uint64_t block_log::get_block_pos(uint32_t block_num) const {
       my->check_index_read();
-
-      if (!(my->head && block_num <= block_header::num_from_id(my->head_id) && block_num > 0))
+      if (!(my->head && block_num <= block_header::num_from_id(my->head_id) && block_num >= my->first_block_num))
          return npos;
-      my->index_stream.seekg(sizeof(uint64_t) * (block_num - 1));
+      my->index_stream.seekg(sizeof(uint64_t) * (block_num - my->first_block_num));
       uint64_t pos;
       my->index_stream.read((char*)&pos, sizeof(pos));
       return pos;
@@ -287,13 +309,21 @@ namespace eosio { namespace chain {
 
       my->block_stream.seekg(-sizeof(pos), std::ios::end);
       my->block_stream.read((char*)&pos, sizeof(pos));
-      return read_block(pos).first;
+      if (pos != npos) {
+         return read_block(pos).first;
+      } else {
+         return {};
+      }
    }
 
    const signed_block_ptr& block_log::head()const {
       return my->head;
    }
 
+   uint32_t block_log::first_block_num() const {
+      return my->first_block_num;
+   }
+
    void block_log::construct_index() {
       ilog("Reconstructing Block Log Index...");
       my->index_stream.close();
@@ -308,7 +338,12 @@ namespace eosio { namespace chain {
       my->block_stream.read((char*)&end_pos, sizeof(end_pos));
       signed_block tmp;
 
-      uint64_t pos = 4; // Skip version which should have already been checked.
+      uint64_t pos = 0;
+      if (my->version == 1) {
+         pos = 4; // Skip version which should have already been checked.
+      } else {
+         pos = 8; // Skip version and first block offset which should have already been checked
+      }
       my->block_stream.seekg(pos);
 
       genesis_state gs;
@@ -361,16 +396,23 @@ namespace eosio { namespace chain {
 
       uint32_t version = 0;
       old_block_stream.read( (char*)&version, sizeof(version) );
-      EOS_ASSERT( version > 0, block_log_exception, "Block log was not setup properly with genesis information." );
-      EOS_ASSERT( version == block_log::supported_version, block_log_unsupported_version,
-                 "Unsupported version of block log. Block log version is ${version} while code supports version ${supported}",
-                 ("version", version)("supported", block_log::supported_version) );
+      EOS_ASSERT( version > 0, block_log_exception, "Block log was not setup properly" );
+      EOS_ASSERT( version >= min_supported_version && version <= max_supported_version, block_log_unsupported_version,
+                 "Unsupported version of block log. Block log version is ${version} while code supports version(s) [${min},${max}]",
+                 ("version", version)("min", block_log::min_supported_version)("max", block_log::max_supported_version) );
+
+      new_block_stream.write( (char*)&version, sizeof(version) );
+
+      uint32_t first_block_num = 1;
+      if (version != 1) {
+         old_block_stream.read ( (char*)&first_block_num, sizeof(first_block_num) );
+         new_block_stream.write( (char*)&first_block_num, sizeof(first_block_num) );
+      }
 
       genesis_state gs;
       fc::raw::unpack(old_block_stream, gs);
 
       auto data = fc::raw::pack( gs );
-      new_block_stream.write( (char*)&version, sizeof(version) );
       new_block_stream.write( data.data(), data.size() );
 
       std::exception_ptr     except_ptr;
@@ -472,10 +514,15 @@ namespace eosio { namespace chain {
 
       uint32_t version = 0;
       block_stream.read( (char*)&version, sizeof(version) );
-      EOS_ASSERT( version > 0, block_log_exception, "Block log was not setup properly with genesis information." );
-      EOS_ASSERT( version == block_log::supported_version, block_log_unsupported_version,
-                 "Unsupported version of block log. Block log version is ${version} while code supports version ${supported}",
-                 ("version", version)("supported", block_log::supported_version) );
+      EOS_ASSERT( version > 0, block_log_exception, "Block log was not setup properly." );
+      EOS_ASSERT( version >= min_supported_version && version <= max_supported_version, block_log_unsupported_version,
+                 "Unsupported version of block log. Block log version is ${version} while code supports version(s) [${min},${max}]",
+                 ("version", version)("min", block_log::min_supported_version)("max", block_log::max_supported_version) );
+
+      uint32_t first_block_num = 1;
+      if (version != 1) {
+         block_stream.read ( (char*)&first_block_num, sizeof(first_block_num) );
+      }
 
       genesis_state gs;
       fc::raw::unpack(block_stream, gs);