Remove SSZ Union usage in BlockHeaderWithProof type

fluffy/database/content_db_migrate_deprecated.nim

@@ -0,0 +1,269 @@
+# Fluffy
+# Copyright (c) 2025 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+import
+  chronicles,
+  metrics,
+  stint,
+  results,
+  stew/ptrops,
+  sqlite3_abi,
+  eth/db/kvstore,
+  eth/db/kvstore_sqlite3,
+  eth/common/[headers_rlp, blocks_rlp, receipts_rlp, transactions_rlp],
+  ../network/history/history_content,
+  ../network/history/history_type_conversions,
+  ../network/state/state_utils,
+  ../network/state/state_content,
+  ../network/history/content/content_values_deprecated
+
+export kvstore_sqlite3
+
+type
+  ContentPair = tuple[contentKey: array[32, byte], contentItem: seq[byte]]
+
+  ContentDBDeprecated* = ref object
+    backend: SqStoreRef
+    kv: KvStoreRef
+    selectAllStmt: SqliteStmt[NoParams, ContentPair]
+    deleteBatchStmt: SqliteStmt[NoParams, void]
+    updateBatchStmt: SqliteStmt[NoParams, void]
+
+func isWithoutProofImpl(content: openArray[byte]): bool =
+  let headerWithProof = decodeSsz(content, BlockHeaderWithProofDeprecated).valueOr:
+    # Leave all other content as it is
+    return false
+
+  if headerWithProof.proof.proofType ==
+      BlockHeaderProofType.historicalHashesAccumulatorProof:
+    false
+  elif headerWithProof.proof.proofType == BlockHeaderProofType.none:
+    true
+  else:
+    false
+
+func isWithoutProof*(
+    ctx: SqliteContext, n: cint, v: SqliteValue
+) {.cdecl, gcsafe, raises: [].} =
+  doAssert(n == 1)
+
+  let
+    ptrs = makeUncheckedArray(v)
+    blob1Len = sqlite3_value_bytes(ptrs[][0])
+
+  if isWithoutProofImpl(makeOpenArray(sqlite3_value_blob(ptrs[][0]), byte, blob1Len)):
+    ctx.sqlite3_result_int(cint 1)
+  else:
+    ctx.sqlite3_result_int(cint 0)
+
+func isWithInvalidEncodingImpl(content: openArray[byte]): bool =
+  let headerWithProof = decodeSsz(content, BlockHeaderWithProofDeprecated).valueOr:
+    # Leave all other content as it is
+    return false
+
+  if headerWithProof.proof.proofType ==
+      BlockHeaderProofType.historicalHashesAccumulatorProof: true else: false
+
+func isWithInvalidEncoding*(
+    ctx: SqliteContext, n: cint, v: SqliteValue
+) {.cdecl, gcsafe, raises: [].} =
+  doAssert(n == 1)
+
+  let
+    ptrs = makeUncheckedArray(v)
+    blobLen = sqlite3_value_bytes(ptrs[][0])
+
+  if isWithInvalidEncodingImpl(
+    makeOpenArray(sqlite3_value_blob(ptrs[][0]), byte, blobLen)
+  ):
+    ctx.sqlite3_result_int(cint 1)
+  else:
+    ctx.sqlite3_result_int(cint 0)
+
+func adjustContentImpl(a: openArray[byte]): seq[byte] =
+  let headerWithProof = decodeSsz(a, BlockHeaderWithProofDeprecated).valueOr:
+    raiseAssert("Should not occur as decoding check is already done")
+
+  let accumulatorProof = headerWithProof.proof.historicalHashesAccumulatorProof
+  let adjustedContent = BlockHeaderWithProof(
+    header: headerWithProof.header,
+    proof: ByteList[MAX_HEADER_PROOF_LENGTH].init(SSZ.encode(accumulatorProof)),
+  )
+
+  SSZ.encode(adjustedContent)
+
+func adjustContent*(
+    ctx: SqliteContext, n: cint, v: SqliteValue
+) {.cdecl, gcsafe, raises: [].} =
+  doAssert(n == 1)
+
+  let
+    ptrs = makeUncheckedArray(v)
+    blobLen = sqlite3_value_bytes(ptrs[][0])
+
+    bytes =
+      adjustContentImpl(makeOpenArray(sqlite3_value_blob(ptrs[][0]), byte, blobLen))
+
+  sqlite3_result_blob(ctx, baseAddr bytes, cint bytes.len, SQLITE_TRANSIENT)
+
+template expectDb(x: auto): untyped =
+  # There's no meaningful error handling implemented for a corrupt database or
+  # full disk - this requires manual intervention, so we'll panic for now
+  x.expect("working database (disk broken/full?)")
+
+## Public calls to get database size, content size and similar.
+
+proc new*(
+    T: type ContentDBDeprecated,
+    path: string,
+    inMemory = false,
+    manualCheckpoint = false,
+): ContentDBDeprecated =
+  let db =
+    if inMemory:
+      SqStoreRef.init("", "fluffy-test", inMemory = true).expect(
+        "working database (out of memory?)"
+      )
+    else:
+      SqStoreRef.init(path, "fluffy", manualCheckpoint = false).expectDb()
+
+  db.createCustomFunction("isWithoutProof", 1, isWithoutProof).expect(
+    "Custom function isWithoutProof creation OK"
+  )
+
+  db.createCustomFunction("isWithInvalidEncoding", 1, isWithInvalidEncoding).expect(
+    "Custom function isWithInvalidEncoding creation OK"
+  )
+
+  db.createCustomFunction("adjustContent", 1, adjustContent).expect(
+    "Custom function adjustContent creation OK"
+  )
+
+  let selectAllStmt =
+    db.prepareStmt("SELECT key, value FROM kvstore", NoParams, ContentPair)[]
+
+  let deleteBatchStmt = db.prepareStmt(
+    "DELETE FROM kvstore WHERE key IN (SELECT key FROM kvstore WHERE isWithoutProof(value) == 1)",
+    NoParams, void,
+  )[]
+
+  let updateBatchStmt = db.prepareStmt(
+    "UPDATE kvstore SET value = adjustContent(value) WHERE key IN (SELECT key FROM kvstore WHERE isWithInvalidEncoding(value) == 1)",
+    NoParams, void,
+  )[]
+
+  let kvStore = kvStore db.openKvStore().expectDb()
+
+  let contentDb = ContentDBDeprecated(
+    kv: kvStore,
+    backend: db,
+    selectAllStmt: selectAllStmt,
+    deleteBatchStmt: deleteBatchStmt,
+    updateBatchStmt: updateBatchStmt,
+  )
+
+  contentDb
+
+template disposeSafe(s: untyped): untyped =
+  if distinctBase(s) != nil:
+    s.dispose()
+    s = typeof(s)(nil)
+
+proc close*(db: ContentDBDeprecated) =
+  db.selectAllStmt.disposeSafe()
+  db.deleteBatchStmt.disposeSafe()
+  db.updateBatchStmt.disposeSafe()
+  discard db.kv.close()
+
+proc deleteAllHeadersWithoutProof*(db: ContentDBDeprecated) =
+  notice "ContentDB migration: deleting all headers without proof"
+  db.deleteBatchStmt.exec().expectDb()
+  notice "ContentDB migration done"
+
+proc updateAllHeadersWithInvalidEncoding*(db: ContentDBDeprecated) =
+  notice "ContentDB migration: updating all headers with invalid encoding"
+  db.updateBatchStmt.exec().expectDb()
+  notice "ContentDB migration done"
+
+proc iterateAllAndCountTypes*(db: ContentDBDeprecated) =
+  ## Ugly debugging call to print out count of content types in case of issues.
+  var
+    contentPair: ContentPair
+    contentTotal = 0
+    contentOldHeaders = 0
+    contentNewHeaders = 0
+    contentBodies = 0
+    contentReceipts = 0
+    contentAccount = 0
+    contentContract = 0
+    contentCode = 0
+    contentOther = 0
+
+  notice "ContentDB type count: iterating over all content"
+  for e in db.selectAllStmt.exec(contentPair):
+    contentTotal.inc()
+    block:
+      let res = decodeSsz(contentPair.contentItem, BlockHeaderWithProofDeprecated)
+      if res.isOk():
+        if decodeRlp(res.value().header.asSeq(), Header).isOk():
+          contentOldHeaders.inc()
+          continue
+    block:
+      let res = decodeSsz(contentPair.contentItem, BlockHeaderWithProof)
+      if res.isOk():
+        if decodeRlp(res.value().header.asSeq(), Header).isOk():
+          contentNewHeaders.inc()
+          continue
+
+    block:
+      let res = decodeSsz(contentPair.contentItem, PortalReceipts)
+      if res.isOk():
+        if fromPortalReceipts(seq[Receipt], res.value()).isOk():
+          contentReceipts.inc()
+          continue
+
+    block:
+      let res = decodeSsz(contentPair.contentItem, PortalBlockBodyShanghai)
+      if res.isOk():
+        if fromPortalBlockBody(BlockBody, res.value()).isOk():
+          contentBodies.inc()
+          continue
+
+    block:
+      let res = decodeSsz(contentPair.contentItem, PortalBlockBodyLegacy)
+      if res.isOk():
+        if fromPortalBlockBody(BlockBody, res.value()).isOk():
+          contentBodies.inc()
+          continue
+
+    block:
+      let res = decodeSsz(contentPair.contentItem, AccountTrieNodeRetrieval)
+      if res.isOk():
+        if rlpDecodeAccountTrieNode(res.value().node).isOk():
+          contentAccount.inc()
+          continue
+
+    block:
+      let res = decodeSsz(contentPair.contentItem, ContractTrieNodeRetrieval)
+      if res.isOk():
+        if rlpDecodeContractTrieNode(res.value().node).isOk():
+          contentContract.inc()
+          continue
+
+    block:
+      let res = decodeSsz(contentPair.contentItem, ContractCodeRetrieval)
+      if res.isOk():
+        contentCode.inc()
+        continue
+
+    contentOther.inc()
+
+  notice "ContentDB type count done: ",
+    contentTotal, contentOldHeaders, contentNewHeaders, contentReceipts, contentBodies,
+    contentAccount, contentContract, contentCode, contentOther

fluffy/network/history/content/content_values.nim

@@ -26,25 +26,15 @@ const
   MAX_WITHDRAWALS_COUNT = MAX_WITHDRAWALS_PER_PAYLOAD
 
   MAX_EPHEMERAL_HEADER_PAYLOAD = 256
+  MAX_HEADER_PROOF_LENGTH* = 1024
 
 type
   ## BlockHeader types
   HistoricalHashesAccumulatorProof* = array[15, Digest]
 
-  BlockHeaderProofType* = enum
-    none = 0x00 # An SSZ Union None
-    historicalHashesAccumulatorProof = 0x01
-
-  BlockHeaderProof* = object
-    case proofType*: BlockHeaderProofType
-    of none:
-      discard
-    of historicalHashesAccumulatorProof:
-      historicalHashesAccumulatorProof*: HistoricalHashesAccumulatorProof
-
   BlockHeaderWithProof* = object
     header*: ByteList[MAX_HEADER_LENGTH] # RLP data
-    proof*: BlockHeaderProof
+    proof*: ByteList[MAX_HEADER_PROOF_LENGTH]
 
   ## Ephemeral BlockHeader list
   EphemeralBlockHeaderList* =
@@ -73,11 +63,3 @@ type
   ## Receipts types
   ReceiptByteList* = ByteList[MAX_RECEIPT_LENGTH] # RLP data
   PortalReceipts* = List[ReceiptByteList, MAX_TRANSACTION_COUNT]
-
-func init*(T: type BlockHeaderProof, proof: HistoricalHashesAccumulatorProof): T =
-  BlockHeaderProof(
-    proofType: historicalHashesAccumulatorProof, historicalHashesAccumulatorProof: proof
-  )
-
-func init*(T: type BlockHeaderProof): T =
-  BlockHeaderProof(proofType: none)

fluffy/network/history/content/content_values_deprecated.nim

@@ -0,0 +1,33 @@
+# fluffy
+# Copyright (c) 2025 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+
+import std/math, nimcrypto/hash, ssz_serialization
+
+export ssz_serialization, hash
+
+const MAX_HEADER_LENGTH = 2 ^ 11 # = 2048
+
+type
+  ## BlockHeader types
+  HistoricalHashesAccumulatorProof* = array[15, Digest]
+
+  BlockHeaderProofType* = enum
+    none = 0x00 # An SSZ Union None
+    historicalHashesAccumulatorProof = 0x01
+
+  BlockHeaderProof* = object
+    case proofType*: BlockHeaderProofType
+    of none:
+      discard
+    of historicalHashesAccumulatorProof:
+      historicalHashesAccumulatorProof*: HistoricalHashesAccumulatorProof
+
+  BlockHeaderWithProofDeprecated* = object
+    header*: ByteList[MAX_HEADER_LENGTH] # RLP data
+    proof*: BlockHeaderProof

fluffy/network/history/history_validation.nim

@@ -50,20 +50,23 @@ func validateHeaderBytes*(
   ok(header)
 
 func verifyBlockHeaderProof*(
-    a: FinishedHistoricalHashesAccumulator, header: Header, proof: BlockHeaderProof
+    a: FinishedHistoricalHashesAccumulator,
+    header: Header,
+    proof: ByteList[MAX_HEADER_PROOF_LENGTH],
 ): Result[void, string] =
-  case proof.proofType
-  of BlockHeaderProofType.historicalHashesAccumulatorProof:
-    a.verifyAccumulatorProof(header, proof.historicalHashesAccumulatorProof)
-  of BlockHeaderProofType.none:
-    if header.isPreMerge():
-      err("Pre merge header requires HistoricalHashesAccumulatorProof")
-    else:
-      # TODO:
-      # Add verification post merge based on historical_roots & historical_summaries
-      # Lets for now no longer accept other headers without a proof and the most
-      # recent ones are now a different type.
-      err("Post merge header proofs not yet activated")
+  let timestamp = Moment.init(header.timestamp.int64, Second)
+
+  if isShanghai(chainConfig, timestamp):
+    # TODO: Add verification post merge based on historical_summaries
+    err("Shanghai block verification not implemented")
+  elif isPoSBlock(chainConfig, header.number):
+    # TODO: Add verification post merge based on historical_roots
+    err("PoS block verification not implemented")
+  else:
+    let accumulatorProof = decodeSsz(proof.asSeq(), HistoricalHashesAccumulatorProof).valueOr:
+      return err("Failed decoding accumulator proof: " & error)
+
+    a.verifyAccumulatorProof(header, accumulatorProof)
 
 func validateCanonicalHeaderBytes*(
     bytes: openArray[byte], id: uint64 | Hash32, a: FinishedHistoricalHashesAccumulator

fluffy/network/history/validation/historical_hashes_accumulator.nim

@@ -1,5 +1,5 @@
 # Nimbus
-# Copyright (c) 2022-2024 Status Research & Development GmbH
+# Copyright (c) 2022-2025 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -198,7 +198,7 @@ func buildHeaderWithProof*(
 
   ok(
     BlockHeaderWithProof(
-      header: ByteList[2048].init(rlp.encode(header)),
-      proof: BlockHeaderProof.init(proof),
+      header: ByteList[MAX_HEADER_LENGTH].init(rlp.encode(header)),
+      proof: ByteList[MAX_HEADER_PROOF_LENGTH].init(SSZ.encode(proof)),
     )
   )