fixes to binary serialization caused by overloaded operators

Author: b1bart

libraries/chain/include/eosio/chain/database_utils.hpp

@@ -62,6 +62,17 @@ namespace eosio { namespace chain {
       }
    };
 
+   template<typename DataStream>
+   DataStream& operator << ( DataStream& ds, const shared_blob& b ) {
+      fc::raw::pack(ds, static_cast<const shared_string&>(b));
+      return ds;
+   }
+
+   template<typename DataStream>
+   DataStream& operator >> ( DataStream& ds, shared_blob& b ) {
+      fc::raw::unpack(ds, static_cast<shared_string &>(b));
+      return ds;
+   }
 } }
 
 namespace fc {
@@ -131,30 +142,18 @@ namespace fc {
       from_variant(v, _v);
       sv = eosio::chain::shared_vector<T>(_v.begin(), _v.end(), sv.get_allocator());
    }
-
-   template<typename DataStream>
-   DataStream& operator << ( DataStream& ds, const eosio::chain::shared_blob& b ) {
-      fc::raw::pack(ds, static_cast<const eosio::chain::shared_string&>(b));
-      return ds;
-   }
-
-   template<typename DataStream>
-   DataStream& operator >> ( DataStream& ds, eosio::chain::shared_blob& b ) {
-      fc::raw::unpack(ds, static_cast<eosio::chain::shared_string &>(b));
-      return ds;
-   }
 }
 
 namespace chainbase {
    // overloads for OID packing
    template<typename DataStream, typename OidType>
-   DataStream& operator << ( DataStream& ds, const chainbase::oid<OidType>& oid ) {
+   DataStream& operator << ( DataStream& ds, const oid<OidType>& oid ) {
       fc::raw::pack(ds, oid._id);
       return ds;
    }
 
    template<typename DataStream, typename OidType>
-   DataStream& operator >> ( DataStream& ds, chainbase::oid<OidType>& oid ) {
+   DataStream& operator >> ( DataStream& ds, oid<OidType>& oid ) {
       fc::raw::unpack(ds, oid._id);
       return ds;
    }
@@ -169,7 +168,7 @@ DataStream& operator << ( DataStream& ds, const float64_t& v ) {
 
 template<typename DataStream>
 DataStream& operator >> ( DataStream& ds, float64_t& v ) {
-   fc::raw::unpack(ds, *reinterpret_cast<const double *>(&v));
+   fc::raw::unpack(ds, *reinterpret_cast<double *>(&v));
    return ds;
 }
 
@@ -181,7 +180,7 @@ DataStream& operator << ( DataStream& ds, const float128_t& v ) {
 
 template<typename DataStream>
 DataStream& operator >> ( DataStream& ds, float128_t& v ) {
-   fc::raw::unpack(ds, *reinterpret_cast<const eosio::chain::uint128_t*>(&v));
+   fc::raw::unpack(ds, *reinterpret_cast<eosio::chain::uint128_t*>(&v));
    return ds;
 }
 

libraries/chain/include/eosio/chain/permission_object.hpp

@@ -4,6 +4,7 @@
  */
 #pragma once
 #include <eosio/chain/authority.hpp>
+#include <eosio/chain/database_utils.hpp>
 
 #include "multi_index_includes.hpp"
 
@@ -112,5 +113,4 @@ CHAINBASE_SET_INDEX_TYPE(eosio::chain::permission_usage_object, eosio::chain::pe
 
 FC_REFLECT(eosio::chain::permission_object, (usage_id)(parent)(owner)(name)(last_updated)(auth))
 
-FC_REFLECT(chainbase::oid<eosio::chain::permission_usage_object>, (_id))
 FC_REFLECT(eosio::chain::permission_usage_object, (id)(last_used))

libraries/chain/include/eosio/chain/snapshot.hpp

@@ -40,17 +40,55 @@ namespace eosio { namespace chain {
          value = row;
       }
 
+      /**
+       * Due to a pattern in our code of overloading `operator << ( std::ostream&, ... )` to provide
+       * human-readable string forms of data, we cannot directly use ostream as those operators will
+       * be used instead of the expected operators.  In otherwords:
+       * fc::raw::pack(fc::datastream...)
+       * will end up calling _very_ different operators than
+       * fc::raw::pack(std::ostream...)
+       */
+      struct ostream_wrapper {
+         explicit ostream_wrapper(std::ostream& s)
+         :inner(s) {
+
+         }
+
+         ostream_wrapper(ostream_wrapper &&) = default;
+         ostream_wrapper(const ostream_wrapper& ) = default;
+
+         auto& write( const char* d, size_t s ) {
+            return inner.write(d, s);
+         }
+
+         auto& put(char c) {
+           return inner.put(c);
+         }
+
+         auto tellp() const {
+            return inner.tellp();
+         }
+
+         auto& seekp(std::ostream::pos_type p) {
+            return inner.seekp(p);
+         }
+
+         std::ostream& inner;
+      };
+
+
       struct abstract_snapshot_row_writer {
-         virtual void write(std::ostream& out) const = 0;
+         virtual void write(ostream_wrapper& out) const = 0;
          virtual variant to_variant() const = 0;
+         virtual std::string row_type_name() const = 0;
       };
 
       template<typename T>
       struct snapshot_row_writer : abstract_snapshot_row_writer {
          explicit snapshot_row_writer( const T& data )
          :data(data) {}
 
-         void write(std::ostream& out) const override {
+         void write(ostream_wrapper& out) const override {
             fc::raw::pack(out, data);
          }
 
@@ -60,6 +98,10 @@ namespace eosio { namespace chain {
             return var;
          }
 
+         std::string row_type_name() const override {
+            return boost::core::demangle( typeid( T ).name() );
+         }
+
          const T& data;
       };
 
@@ -110,6 +152,7 @@ namespace eosio { namespace chain {
       struct abstract_snapshot_row_reader {
          virtual void provide(std::istream& in) const = 0;
          virtual void provide(const fc::variant&) const = 0;
+         virtual std::string row_type_name() const = 0;
       };
 
       template<typename T>
@@ -125,6 +168,10 @@ namespace eosio { namespace chain {
             fc::from_variant(var, data);
          }
 
+         std::string row_type_name() const override {
+            return boost::core::demangle( typeid( T ).name() );
+         }
+
          T& data;
       };
 
@@ -231,11 +278,10 @@ namespace eosio { namespace chain {
          static const uint32_t magic_number = 0x30510550;
 
       private:
-
-         std::ostream&  snapshot;
-         std::streampos header_pos;
-         std::streampos section_pos;
-         uint64_t       row_count;
+         detail::ostream_wrapper snapshot;
+         std::streampos          header_pos;
+         std::streampos          section_pos;
+         uint64_t                row_count;
 
    };
 

libraries/chain/snapshot.cpp

@@ -263,6 +263,9 @@ void istream_snapshot_reader::set_section( const string& section_name ) {
       if (match && snapshot.get() == 0) {
          cur_row = 0;
          num_rows = row_count;
+
+         // leave the stream at the right point
+         restore_pos.cancel();
          return;
       }
    }

libraries/fc

@@ -67,49 +67,82 @@ struct variant_snapshot_suite {
    using writer_t = variant_snapshot_writer;
    using reader_t = variant_snapshot_reader;
    using write_storage_t = fc::mutable_variant_object;
-   using read_storage_t = fc::variant;
+   using snapshot_t = fc::variant;
+
+   struct writer : public writer_t {
+      writer( const std::shared_ptr<write_storage_t>& storage )
+      :writer_t(*storage)
+      ,storage(storage)
+      {
+
+      }
+
+      std::shared_ptr<write_storage_t> storage;
+   };
+
+   struct reader : public reader_t {
+      explicit reader(const snapshot_t& storage)
+      :reader_t(storage)
+      {}
+   };
+
+
+   static auto get_writer() {
+      return std::make_shared<writer>(std::make_shared<write_storage_t>());
+   }
+
+   static auto finalize(const std::shared_ptr<writer>& w) {
+      w->finalize();
+      return snapshot_t(*w->storage);
+   }
+
+   static auto get_reader( const snapshot_t& buffer) {
+      return std::make_shared<reader>(buffer);
+   }
+
 };
 
 struct buffered_snapshot_suite {
    using writer_t = ostream_snapshot_writer;
    using reader_t = istream_snapshot_reader;
-   using write_storage_t = std::stringstream;
-   using read_storage_t = write_storage_t;
-
-};
+   using write_storage_t = std::ostringstream;
+   using snapshot_t = std::string;
+   using read_storage_t = std::istringstream;
 
-template<typename SUITE>
-struct suite_funcs {
-   struct writer : public SUITE::writer_t {
-      writer( const std::shared_ptr<typename SUITE::write_storage_t>& storage )
-      :SUITE::writer_t(*storage)
+   struct writer : public writer_t {
+      writer( const std::shared_ptr<write_storage_t>& storage )
+      :writer_t(*storage)
       ,storage(storage)
       {
 
       }
 
-      std::shared_ptr<typename SUITE::write_storage_t> storage;
+      std::shared_ptr<write_storage_t> storage;
    };
 
-   struct reader : public SUITE::reader_t {
-      explicit reader(typename SUITE::read_storage_t& buffer)
-      :SUITE::reader_t(buffer)
+   struct reader : public reader_t {
+      explicit reader(const std::shared_ptr<read_storage_t>& storage)
+      :reader_t(*storage)
+      ,storage(storage)
       {}
 
+      std::shared_ptr<read_storage_t> storage;
    };
 
+
    static auto get_writer() {
-      return std::make_shared<writer>(std::make_shared<typename SUITE::write_storage_t>());
+      return std::make_shared<writer>(std::make_shared<write_storage_t>());
    }
 
    static auto finalize(const std::shared_ptr<writer>& w) {
       w->finalize();
-      return typename SUITE::read_storage_t(std::move(*w->storage));
+      return w->storage->str();
    }
 
-   static auto get_reader( typename SUITE::read_storage_t& buffer) {
-      return std::make_shared<reader>(buffer);
+   static auto get_reader( const snapshot_t& buffer) {
+      return std::make_shared<reader>(std::make_shared<read_storage_t>(buffer));
    }
+
 };
 
 BOOST_AUTO_TEST_SUITE(snapshot_tests)
@@ -132,12 +165,12 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_exhaustive_snapshot, SNAPSHOT_SUITE, snapshot
 
    for (int generation = 0; generation < generation_count; generation++) {
       // create a new snapshot child
-      auto writer = suite_funcs<SNAPSHOT_SUITE>::get_writer();
+      auto writer = SNAPSHOT_SUITE::get_writer();
       chain.control->write_snapshot(writer);
-      auto snapshot = suite_funcs<SNAPSHOT_SUITE>::finalize(writer);
+      auto snapshot = SNAPSHOT_SUITE::finalize(writer);
 
       // create a new child at this snapshot
-      sub_testers.emplace_back(chain.get_config(), suite_funcs<SNAPSHOT_SUITE>::get_reader(snapshot), generation);
+      sub_testers.emplace_back(chain.get_config(), SNAPSHOT_SUITE::get_reader(snapshot), generation);
 
       // increment the test contract
       chain.push_action(N(snapshot), N(increment), N(snapshot), mutable_variant_object()
@@ -188,12 +221,12 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_replay_over_snapshot, SNAPSHOT_SUITE, snapsho
    auto expected_pre_integrity_hash = chain.control->calculate_integrity_hash();
 
    // create a new snapshot child
-   auto writer = suite_funcs<SNAPSHOT_SUITE>::get_writer();
+   auto writer = SNAPSHOT_SUITE::get_writer();
    chain.control->write_snapshot(writer);
-   auto snapshot = suite_funcs<SNAPSHOT_SUITE>::finalize(writer);
+   auto snapshot = SNAPSHOT_SUITE::finalize(writer);
 
    // create a new child at this snapshot
-   snapshotted_tester snap_chain(chain.get_config(), suite_funcs<SNAPSHOT_SUITE>::get_reader(snapshot), 1);
+   snapshotted_tester snap_chain(chain.get_config(), SNAPSHOT_SUITE::get_reader(snapshot), 1);
    BOOST_REQUIRE_EQUAL(expected_pre_integrity_hash.str(), snap_chain.control->calculate_integrity_hash().str());
 
    // push more blocks to build up a block log
@@ -213,7 +246,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_replay_over_snapshot, SNAPSHOT_SUITE, snapsho
    BOOST_REQUIRE_EQUAL(expected_post_integrity_hash.str(), snap_chain.control->calculate_integrity_hash().str());
 
    // replay the block log from the snapshot child, from the snapshot
-   snapshotted_tester replay_chain(chain.get_config(), suite_funcs<SNAPSHOT_SUITE>::get_reader(snapshot), 2, 1);
+   snapshotted_tester replay_chain(chain.get_config(), SNAPSHOT_SUITE::get_reader(snapshot), 2, 1);
    BOOST_REQUIRE_EQUAL(expected_post_integrity_hash.str(), snap_chain.control->calculate_integrity_hash().str());
 }