trial hack

Author: gramseyer

memory_database/user_account.h

@@ -35,6 +35,9 @@ class UserAccount {
 	static_assert(
 		__builtin_popcount(MAX_OPS_PER_TX) == 1, "should be power of two");
 
+	static_assert(
+		MAX_SEQ_NUMS_PER_BLOCK <= 64, "rework sequence num reservations");
+
 	using amount_t = typename RevertableAsset::amount_t;
 
 	mutable std::mutex uncommitted_assets_mtx;

modlog/log_entry_fns.cc

@@ -14,9 +14,9 @@ struct NewSelfTransactionCompareFn {
 	}
 };
 
-template<typename Value, typename CompareFn>
-void dedup(std::vector<Value>& values, CompareFn comparator) {
-	for (std::size_t i = 1; i < values.size(); i++) {
+template<typename value_list, typename CompareFn>
+void dedup(value_list& values, CompareFn comparator) {
+	for (size_t i = 1u; i < values.size(); i++) {
 		if (comparator(values[i], values[i-1])) {
 			values.erase(values.begin() + i);
 		} else {
@@ -25,6 +25,33 @@ void dedup(std::vector<Value>& values, CompareFn comparator) {
 	}
 }
 
+void 
+LogMergeFn::value_merge(
+	AccountModificationTxList& original_value, 
+	const AccountModificationTxList& merge_in_value)
+{
+	for (const auto& new_tx : merge_in_value.new_transactions_self) {
+		original_value.new_transactions_self.push_back(new_tx);
+	}
+	//original_value.new_transactions_self.insert(
+	//	original_value.new_transactions_self.end(),
+	//	merge_in_value.new_transactions_self.begin(),
+	//	merge_in_value.new_transactions_self.end());
+	original_value.identifiers_self.insert(
+		original_value.identifiers_self.end(),
+		merge_in_value.identifiers_self.begin(),
+		merge_in_value.identifiers_self.end());
+	original_value.identifiers_others.insert(
+		original_value.identifiers_others.end(),
+		merge_in_value.identifiers_others.begin(),
+		merge_in_value.identifiers_others.end());
+
+	if (original_value.owner != merge_in_value.owner) {
+		throw std::runtime_error("owner mismatch when merging logs!!!");
+	}
+}
+
+
 void 
 LogNormalizeFn::apply_to_value (AccountModificationTxListWrapper& log) {
 	std::sort(log.identifiers_self.begin(), log.identifiers_self.end());

modlog/log_entry_fns.h

@@ -24,11 +24,11 @@ struct LogInsertFn : public GenericInsertFn {
 
 	static void value_insert(AccountModificationTxList& main_value, const SignedTransaction& self_transaction) {
 		main_value.new_transactions_self.push_back(self_transaction);
-		if (main_value.new_transactions_self.size() > main_value.new_transactions_self.capacity() || main_value.new_transactions_self.size() > 500000) {
-			std::printf("%lu %lu %lu\n", main_value.new_transactions_self.size(), main_value.new_transactions_self.capacity(),
-				main_value.new_transactions_self.size());
-			throw std::runtime_error("invalid main_value!!!");
-		}
+		//if (main_value.new_transactions_self.size() > main_value.new_transactions_self.capacity() || main_value.new_transactions_self.size() > 500000) {
+		//	std::printf("%lu %lu %lu\n", main_value.new_transactions_self.size(), main_value.new_transactions_self.capacity(),
+		//		main_value.new_transactions_self.size());
+		//	throw std::runtime_error("invalid main_value!!!");
+		//}
 	}
 /*
 	template<typename AtomicMetadataType, typename ValueType>
@@ -49,24 +49,9 @@ struct LogInsertFn : public GenericInsertFn {
 
 //might like to make these one struct, reduce extra code/etc, but type signatures on metadata merge are slightly diff
 struct LogMergeFn {
-	static void value_merge(AccountModificationTxList& original_value, const AccountModificationTxList& merge_in_value) {
-		original_value.new_transactions_self.insert(
-			original_value.new_transactions_self.end(),
-			merge_in_value.new_transactions_self.begin(),
-			merge_in_value.new_transactions_self.end());
-		original_value.identifiers_self.insert(
-			original_value.identifiers_self.end(),
-			merge_in_value.identifiers_self.begin(),
-			merge_in_value.identifiers_self.end());
-		original_value.identifiers_others.insert(
-			original_value.identifiers_others.end(),
-			merge_in_value.identifiers_others.begin(),
-			merge_in_value.identifiers_others.end());
-
-		if (original_value.owner != merge_in_value.owner) {
-			throw std::runtime_error("owner mismatch when merging logs!!!");
-		}
-	}
+	static void value_merge(
+		AccountModificationTxList& original_value, 
+		const AccountModificationTxList& merge_in_value);
 };
 
 struct LogNormalizeFn {

speedex/speedex_persistence.h

@@ -24,6 +24,10 @@ Phase 3: Everything else (orderbooks, header hash)
 
 namespace speedex {
 
+/*! Call before sending transaction block to a validator.
+Persists account block + header, and prepares memory database with a persistence
+thunk.
+*/
 std::unique_ptr<AccountModificationBlock>
 persist_critical_round_data(
 	SpeedexManagementStructures& management_structures,
@@ -32,24 +36,28 @@ persist_critical_round_data(
 	bool get_block = false,
 	uint64_t log_offset = 0);
 
+//! Memory database loads persistence thunk into lmdb
 void 
 persist_async_phase1(
 	SpeedexManagementStructures& management_structures,
 	const uint64_t current_block_number,
 	BlockDataPersistenceMeasurements& measurements);
 
+//! Msync the memory database lmdb
 void
 persist_async_phase2(
 	SpeedexManagementStructures& management_structures,
 	uint64_t current_block_number,
 	BlockDataPersistenceMeasurements& measurements);
 
+//! Finish persistence (orderbooks, header hash map).
 void
 persist_async_phase3(
 	SpeedexManagementStructures& management_structures,
 	uint64_t current_block_number,
 	BlockDataPersistenceMeasurements& measurements);
 
+//! Operates a background thread for phase 3 persistence.
 class AsyncPersisterPhase3 : public AsyncWorker {
 	using AsyncWorker::mtx;
 	using AsyncWorker::cv;
@@ -90,7 +98,8 @@ class AsyncPersisterPhase3 : public AsyncWorker {
 	}
 };
 
-
+//! Operates a background thread for phase 2 persistence.
+//! Automatically calls phase 3 when done.
 class AsyncPersisterPhase2 : public AsyncWorker {
 	using AsyncWorker::mtx;
 	using AsyncWorker::cv;
@@ -137,6 +146,8 @@ class AsyncPersisterPhase2 : public AsyncWorker {
 	}
 };
 
+//! Operates a background thread for phase 1 persistence.
+//! Automatically calls phase 2 when done.
 struct AsyncPersister : public AsyncWorker {
 	using AsyncWorker::mtx;
 	using AsyncWorker::cv;
@@ -157,7 +168,6 @@ struct AsyncPersister : public AsyncWorker {
 		return latest_measurements != nullptr;
 	}
 
-
 	void run();
 
 public:
@@ -169,13 +179,15 @@ struct AsyncPersister : public AsyncWorker {
 			start_async_thread([this] {run();});
 		}
 
-
-
 	uint64_t get_highest_persisted_block() {
 		std::lock_guard lock(mtx);
 		return highest_persisted_block;
 	}
 
+
+	//! Begin persisting a block to disk 
+	//! (all blocks up to persist_block_number).
+	//! When phase 1 finishes, phase 2 is automatically called.
 	void do_async_persist(
 		const uint64_t persist_block_number, 
 		BlockDataPersistenceMeasurements& measurements) {
@@ -197,6 +209,10 @@ struct AsyncPersister : public AsyncWorker {
 		cv.notify_one();
 	}
 
+	//! Wait for all async persistence phases to complete.
+	//! Clears up all uses of measurements object reference.
+	//! Should be called before shutdown or before invalidating measurements
+	//! object reference.
 	void wait_for_async_persist() {
 		//clears up all uses of measurements reference
 		wait_for_async_task();

trie/prefix.h

@@ -1,5 +1,11 @@
 #pragma once
 
+
+/*! \file prefix.h
+
+Two implementations of a trie prefix.  One is an arbitrary-length
+byte array, and one is specialized for accountid keys.
+*/
 #include <atomic>
 #include <compare>
 #include <concepts>
@@ -367,6 +373,7 @@ class AccountIDPrefix {
 	bool 
 	operator==(const AccountIDPrefix& other) const = default;
 
+	//! Get the bits of the prefix just beyond branch_point
 	unsigned char get_branch_bits(const PrefixLenBits branch_point) const {
 		if (branch_point.len >= MAX_LEN_BITS) {
 			std::printf("Bad branch bits was %u\n", branch_point.len);
@@ -375,6 +382,8 @@ class AccountIDPrefix {
 		return (prefix >> (60 - branch_point.len)) & BRANCH_MASK;
 	}
 
+	//! Compute the length of the longest matching initial subsequence
+	//! of this prefix and the other prefix.
 	PrefixLenBits get_prefix_match_len(
 		const PrefixLenBits& self_len, 
 		const AccountIDPrefix& other, 
@@ -392,10 +401,12 @@ class AccountIDPrefix {
 		return std::min({computed, self_len, other_len});
 	}
 
+	//! Truncate the prefix to a defined length
 	void truncate(const PrefixLenBits truncate_point) {
 		prefix &= (UINT64_MAX << (64 - truncate_point.len));
 	}
 
+	//! Convert prefix to an array of bytes.
 	xdr::opaque_array<MAX_LEN_BYTES> get_bytes_array() const {
 		xdr::opaque_array<MAX_LEN_BYTES> out;
 		write_unsigned_big_endian(out, prefix);
@@ -419,6 +430,7 @@ class AccountIDPrefix {
 		return debug::array_to_str(bytes.data(), len.num_prefix_bytes());
 	}
 
+	//! Modify the prefix by setting the bits after fixed_len to bb
 	void set_next_branch_bits(PrefixLenBits fixed_len, const uint8_t bb) {
 		uint8_t offset = (60-fixed_len.len);	
 		uint64_t mask = ((uint64_t) BRANCH_MASK) << offset;

trie/recycling_impl/trie.h

@@ -446,6 +446,9 @@ class SerialAccountTrie {
 
 	friend class AccountTrie<ValueType>;
 
+	static_assert(sizeof(node_t) <= 64, "account trie node should be at most 1 cache line");
+
+
 public:
 
 	SerialAccountTrie(AccountTrieNodeAllocator<node_t>& allocator) 

utils/save_load_xdr.h

@@ -273,7 +273,7 @@ save_account_block_fast(
 
 	size_t total_written_bytes = 0;
 
-	const xdr::xvector<SignedTransaction>* tx_buffer = nullptr;
+	const xdr::xvector<SignedTransaction, MAX_SEQ_NUMS_PER_BLOCK>* tx_buffer = nullptr;
 	size_t buffer_idx = 0;
 	size_t num_written = 0;
 

xdr/database_commitments.x

@@ -42,7 +42,7 @@ struct TxIdentifier {
 
 struct AccountModificationTxList {
 	AccountID owner;
-	SignedTransaction new_transactions_self<>; //transactions
+	SignedTransaction new_transactions_self<MAX_SEQ_NUMS_PER_BLOCK>; //transactions
 	uint64 identifiers_self<>; // sequence numbers, in addition
 	TxIdentifier identifiers_others<>;
 };

xdr/transaction.x

@@ -106,6 +106,7 @@ struct Operation {
 
 const MAX_OPS_PER_TX = 256;
 const RESERVED_SEQUENCE_NUM_LOWBITS = 255;
+const MAX_SEQ_NUMS_PER_BLOCK = 64;
 
 struct TransactionMetadata {
 	AccountID sourceAccount;