Portal + era1: Remove use of hardcoded mainnet mergeBlockNumber (#3742)

Make it configurable in historical_hashes_accumulator and era1
code. This way it can be used for different networks (testnets).

As a result we also don't have to separately compile the
historical_hashes_accumulator tests with a different compile
time define.

Also tested nimbus_execution_client import for Sepolia in the
process and found an issue with the era1 files there which is
why an additional check in era1 open call got commented out for now.

Additional things that came up:
- Time to bring the two eraDBs together (Portal + EL).
- Time to bring Portal and EL ChainConfig together but the latter
could use some cleanup

Author: kdeme

Makefile

@@ -290,17 +290,13 @@ portal-test-reproducibility:
 		[ "$$MD5SUM1" = "$$MD5SUM2" ] && echo -e "\e[92mSuccess: identical binaries.\e[39m" || \
 			{ echo -e "\e[91mFailure: the binary changed between builds.\e[39m"; exit 1; }
 
-# Portal tests
-all_eth_history_custom_chain_tests: | build deps
-	echo -e $(BUILD_MSG) "build/$@" && \
-	$(ENV_SCRIPT) nim c -r $(NIM_PARAMS) -d:chronicles_log_level=ERROR -d:mergeBlockNumber:38130 -o:build/$@ "portal/tests/eth_history_tests/$@.nim"
-
+# builds and runs the Portal test suite
 all_portal_tests: | build deps
 	echo -e $(BUILD_MSG) "build/$@" && \
 	$(ENV_SCRIPT) nim c -r $(NIM_PARAMS) -d:chronicles_log_level=ERROR -o:build/$@ "portal/tests/$@.nim"
 
-# builds and runs the Portal test suite
-portal-test: | all_portal_tests all_eth_history_custom_chain_tests
+# alias for all_portal_tests
+portal-test: | all_portal_tests
 
 # builds the Portal tools, wherever they are
 $(PORTAL_TOOLS): | build deps rocksdb

execution_chain/db/era1_db/db_desc.nim

@@ -30,6 +30,7 @@ type Era1DbRef* = ref object
   network: string
   files: seq[Era1File]
   filenames: Table[uint64, string]
+  mergeBlockNumber: uint64
 
 proc getEra1File*(db: Era1DbRef, era: Era1): Result[Era1File, string] =
   for f in db.files:
@@ -49,7 +50,7 @@ proc getEra1File*(db: Era1DbRef, era: Era1): Result[Era1File, string] =
 
   # TODO: The open call does not do full verification. It is assumed here that
   # trusted files are used. We might want to add a full validation option.
-  let f = Era1File.open(path).valueOr:
+  let f = Era1File.open(path, db.mergeBlockNumber).valueOr:
     return err(path & ": " & error)
 
   if db.files.len > 16: # TODO LRU
@@ -60,7 +61,7 @@ proc getEra1File*(db: Era1DbRef, era: Era1): Result[Era1File, string] =
   ok(f)
 
 proc init*(
-    T: type Era1DbRef, path: string, network: string
+    T: type Era1DbRef, path: string, network: string, mergeBlockNumber: uint64
 ): Result[Era1DbRef, string] =
   var filenames: Table[uint64, string]
   try:
@@ -78,7 +79,7 @@ proc init*(
   if filenames.len == 0:
     return err "No era files found in " & path
 
-  ok Era1DbRef(path: path, network: network, filenames: filenames)
+  ok Era1DbRef(path: path, network: network, filenames: filenames, mergeBlockNumber: mergeBlockNumber)
 
 proc getEthBlock*(
     db: Era1DbRef, blockNumber: uint64, res: var Block

execution_chain/nimbus_import.nim

@@ -275,7 +275,7 @@ proc importBlocks*(conf: NimbusConf, com: CommonRef) =
           "sepolia"
         else:
           raiseAssert "Other networks are unsupported or do not have an era1"
-      db = Era1DbRef.init(conf.era1Dir, era1Name).valueOr:
+      db = Era1DbRef.init(conf.era1Dir, era1Name, lastEra1Block + 1).valueOr:
         fatal "Could not open era1 database",
           era1Dir = conf.era1Dir, era1Name = era1Name, error = error
         quit(QuitFailure)

portal/bridge/history/portal_history_bridge.nim

@@ -22,18 +22,21 @@ import
   ../../rpc/portal_rpc_client,
   ../../network/history/[history_content, history_validation],
   ../../eth_history/block_proofs/historical_hashes_accumulator,
-  ../../network/network_metadata,
   ../../eth_history/[era1, history_data_ssz_e2s],
   ../../database/era1_db,
+  ../../../execution_chain/common/[hardforks, chain_config],
   ../common/rpc_helpers,
   ../nimbus_portal_bridge_conf
 
+from ../../network/network_metadata import loadAccumulator
+
 const newHeadPollInterval = 6.seconds # Slot with potential block is every 12s
 
 type PortalHistoryBridge = ref object
   portalClient: RpcClient
   web3Client: RpcClient
   gossipQueue: AsyncQueue[(seq[byte], seq[byte])]
+  cfg*: ChainConfig
 
 proc gossipBlockBody(
     bridge: PortalHistoryBridge, blockNumber: uint64, body: BlockBody
@@ -106,7 +109,7 @@ proc runLatestLoop(
 proc gossipBlockContent(
     bridge: PortalHistoryBridge, era1File: string, verifyEra = false
 ): Future[Result[void, string]] {.async: (raises: [CancelledError]).} =
-  let f = ?Era1File.open(era1File)
+  let f = ?Era1File.open(era1File, bridge.cfg.posBlock.get())
 
   if verifyEra:
     let _ = ?f.verify()
@@ -143,7 +146,7 @@ proc runBackfillLoopAuditMode(
 ) {.async: (raises: [CancelledError]).} =
   let
     rng = newRng()
-    db = Era1DB.new(era1Dir, "mainnet", loadAccumulator())
+    db = Era1DB.new(era1Dir, "mainnet", loadAccumulator(), bridge.cfg.posBlock.get())
     blockLowerBound = startEra * EPOCH_SIZE # inclusive
     blockUpperBound = ((endEra + 1) * EPOCH_SIZE) - 1 # inclusive
     blockRange = blockUpperBound - blockLowerBound
@@ -207,6 +210,7 @@ proc runHistory*(config: PortalBridgeConf) =
     portalClient: newRpcClientConnect(config.portalRpcUrl),
     web3Client: newRpcClientConnect(config.web3Url),
     gossipQueue: newAsyncQueue[(seq[byte], seq[byte])](config.gossipConcurrency),
+    cfg: chainConfigForNetwork(MainNet),
   )
 
   proc gossipWorker(bridge: PortalHistoryBridge) {.async: (raises: []).} =

portal/database/era1_db.nim

@@ -22,13 +22,14 @@ type Era1DB* = ref object
   network: string
   accumulator: FinishedHistoricalHashesAccumulator
   files: seq[Era1File]
+  mergeBlockNumber: uint64
 
 proc getEra1File(db: Era1DB, era: Era1): Result[Era1File, string] =
   for f in db.files:
     if f.blockIdx.startNumber.era == era:
       return ok(f)
 
-  if era > mergeBlockNumber.era():
+  if era > db.mergeBlockNumber.era():
     return err("Selected era1 past pre-merge data")
 
   let
@@ -41,7 +42,7 @@ proc getEra1File(db: Era1DB, era: Era1): Result[Era1File, string] =
 
   # TODO: The open call does not do full verification. It is assumed here that
   # trusted files are used. We might want to add a full validation option.
-  let f = Era1File.open(path).valueOr:
+  let f = Era1File.open(path, db.mergeBlockNumber).valueOr:
     return err(error)
 
   if db.files.len > 16: # TODO LRU
@@ -56,8 +57,15 @@ proc new*(
     path: string,
     network: string,
     accumulator: FinishedHistoricalHashesAccumulator,
+    mergeBlockNumber: uint64,
 ): Era1DB =
-  Era1DB(path: path, network: network, accumulator: accumulator)
+  # TODO: Calculate mergeBlockNumber from accumulator instead.
+  Era1DB(
+    path: path,
+    network: network,
+    accumulator: accumulator,
+    mergeBlockNumber: mergeBlockNumber,
+  )
 
 proc getEthBlock*(
     db: Era1DB, blockNumber: uint64, res: var Block

portal/eth_history/block_proofs/block_proof_historical_hashes_accumulator.nim

@@ -12,10 +12,12 @@ import
   ssz_serialization,
   ssz_serialization/[proofs, merkleization],
   ../../common/common_types,
+  ../../network/network_metadata,
   ./historical_hashes_accumulator
 
 export
-  ssz_serialization, merkleization, proofs, common_types, historical_hashes_accumulator
+  ssz_serialization, merkleization, proofs, common_types, historical_hashes_accumulator,
+  network_metadata
 
 #
 # Implementation of pre-merge block proofs by making use of the frozen HistoricalHashesAccumulator
@@ -60,12 +62,6 @@ func getHeaderRecordIndex*(header: Header, epochIndex: uint64): uint64 =
   ## Get the relative header index for the epoch accumulator
   getHeaderRecordIndex(header.number, epochIndex)
 
-func isPreMerge*(blockNumber: uint64): bool =
-  blockNumber < mergeBlockNumber
-
-func isPreMerge*(header: Header): bool =
-  isPreMerge(header.number)
-
 func verifyProof*(
     a: FinishedHistoricalHashesAccumulator,
     header: Header,
@@ -95,9 +91,11 @@ func verifyProof*(
   verify_merkle_multiproof(@[leave], proof, @[gIndex], epochRecordHash)
 
 func buildProof*(
-    header: Header, epochRecord: EpochRecord | EpochRecordCached
+    chainConfig: ChainConfig,
+    header: Header,
+    epochRecord: EpochRecord | EpochRecordCached,
 ): Result[HistoricalHashesAccumulatorProof, string] =
-  doAssert(header.isPreMerge(), "Must be pre merge header")
+  doAssert(not chainConfig.isPoSBlock(header), "Must be pre merge header")
 
   let
     epochIndex = getEpochIndex(header)

portal/eth_history/block_proofs/historical_hashes_accumulator.nim

@@ -27,18 +27,6 @@ const
   MAX_HISTORICAL_EPOCHS = 2048'u64 # Should be sufficient for all networks as for
   # mainnet this is not even reached: ceil(mergeBlockNumber / EPOCH_SIZE) = 1897
 
-  # Allow this to be adjusted at compile time for testing. If more constants
-  # need to be adjusted we can add some presets file.
-  mergeBlockNumber* {.intdefine.}: uint64 = 15537394
-
-  # Note: This is like a ceil(mergeBlockNumber / EPOCH_SIZE)
-  # Could use ceilDiv(mergeBlockNumber, EPOCH_SIZE) in future versions
-  preMergeEpochs* = (mergeBlockNumber + EPOCH_SIZE - 1) div EPOCH_SIZE
-
-  # TODO:
-  # Currently disabled, because issue when testing with other
-  # `mergeBlockNumber`, but it could be used as value to double check on at
-  # merge block.
   # TODO: Could also be used as value to actual finish the accumulator, instead
   # of `mergeBlockNumber`, but:
   # - Still need to store the actual `mergeBlockNumber` and run-time somewhere
@@ -84,10 +72,6 @@ func getEpochRecordRoot*(headerRecords: openArray[HeaderRecord]): Digest =
   hash_tree_root(epochRecord)
 
 func updateAccumulator*(a: var HistoricalHashesAccumulator, header: Header) =
-  doAssert(
-    header.number < mergeBlockNumber, "No post merge blocks for header accumulator"
-  )
-
   let lastTotalDifficulty =
     if a.currentEpoch.len() == 0:
       0.stuint(256)

portal/eth_history/era1.nim

@@ -156,7 +156,7 @@ proc skipRecord*(f: IoHandle): Result[void, string] =
 func startNumber*(era: Era1): uint64 =
   era * MaxEra1Size
 
-func endNumber*(era: Era1): uint64 =
+func endNumber*(era: Era1, mergeBlockNumber: uint64): uint64 =
   if (era + 1) * MaxEra1Size - 1'u64 >= mergeBlockNumber:
     # The incomplete era just before the merge
     mergeBlockNumber - 1'u64
@@ -169,7 +169,7 @@ func endNumber*(blockIdx: BlockIndex): uint64 =
 func era*(blockNumber: uint64): Era1 =
   Era1(blockNumber div MaxEra1Size)
 
-func offsetsLen(startNumber: uint64): int =
+func offsetsLen(startNumber: uint64, mergeBlockNumber: uint64): int =
   # For the era where the merge happens the era files only holds the blocks
   # until the merge block so the offsets length needs to be adapted too.
   if startNumber.era() >= mergeBlockNumber.era():
@@ -187,13 +187,16 @@ proc fromCompressedRlpBytes[T](bytes: openArray[byte], v: var T): Result[void, s
   except RlpError as e:
     err("Invalid compressed RLP data for " & $T & ": " & e.msg)
 
-proc init*(T: type Era1Group, f: IoHandle, startNumber: uint64): Result[T, string] =
+proc init*(
+    T: type Era1Group, f: IoHandle, startNumber: uint64, mergeBlockNumber: uint64
+): Result[T, string] =
   discard ?f.appendHeader(E2Version, 0)
 
   ok(
     Era1Group(
       blockIndex: BlockIndex(
-        startNumber: startNumber, offsets: newSeq[int64](startNumber.offsetsLen())
+        startNumber: startNumber,
+        offsets: newSeq[int64](offsetsLen(startNumber, mergeBlockNumber)),
       )
     )
   )
@@ -268,7 +271,9 @@ type
   BlockTuple* =
     tuple[header: headers.Header, body: BlockBody, receipts: seq[Receipt], td: UInt256]
 
-proc open*(_: type Era1File, name: string): Result[Era1File, string] =
+proc open*(
+    _: type Era1File, name: string, mergeBlockNumber: uint64
+): Result[Era1File, string] =
   var f = Opt[IoHandle].ok(?openFile(name, {OpenFlags.Read}).mapErr(ioErrorMsg))
 
   defer:
@@ -284,8 +289,11 @@ proc open*(_: type Era1File, name: string): Result[Era1File, string] =
   ?f[].setFilePos(blockIdxPos, SeekPosition.SeekCurrent).mapErr(ioErrorMsg)
 
   let blockIdx = ?f[].readBlockIndex()
-  if blockIdx.offsets.len() != blockIdx.startNumber.offsetsLen():
-    return err("Block index length invalid")
+  # TODO: Re-enable this check when Sepolia era files are fixed
+  # let offsetLen = offsetsLen(blockIdx.startNumber, mergeBlockNumber)
+  # if blockIdx.offsets.len() != offsetLen:
+  #   return err("Block index length invalid: " & $blockIdx.offsets.len() & " vs expected " &
+  #     $offsetLen)
 
   let res = Era1File(handle: f, blockIdx: blockIdx)
   reset(f)

portal/network/network_metadata.nim

@@ -12,6 +12,7 @@ import
   results,
   stew/io2,
   chronos/timer,
+  eth/common/headers,
   beacon_chain/spec/forks,
   ../eth_history/block_proofs/historical_hashes_accumulator
 
@@ -95,6 +96,9 @@ func isTimestampForked(forkTime: Opt[Moment], timestamp: Moment): bool =
 func isPoSBlock*(c: ChainConfig, blockNumber: uint64): bool =
   c.mergeNetsplitBlock <= blockNumber
 
+func isPoSBlock*(c: ChainConfig, header: Header): bool =
+  c.mergeNetsplitBlock <= header.number
+
 func isShanghai*(c: ChainConfig, timestamp: Moment): bool =
   isTimestampForked(c.shanghaiTime, timestamp)