fixes some basic decoding and padding mistakes

Author: marcinczenko

codex/bittorrent/manifest/decoding.nim

@@ -6,7 +6,7 @@ import pkg/questionable/results
 import ../../blocktype
 import ./manifest
 
-proc decode(_: type BitTorrentManifest, data: openArray[byte]): ?!BitTorrentManifest =
+func decode(_: type BitTorrentManifest, data: openArray[byte]): ?!BitTorrentManifest =
   # ```protobuf
   #   Message BitTorrentManifest {
   #     Message Piece {
@@ -25,7 +25,7 @@ proc decode(_: type BitTorrentManifest, data: openArray[byte]): ?!BitTorrentMani
   # ```
 
   var
-    pbNode = initProtoBuffer()
+    pbNode = initProtoBuffer(data)
     pbInfo: ProtoBuffer
     length: uint64
     pieceLength: uint32
@@ -50,7 +50,7 @@ proc decode(_: type BitTorrentManifest, data: openArray[byte]): ?!BitTorrentMani
       var dataBuf = newSeq[byte]()
       if pbPiece.getField(1, dataBuf).isErr:
         return failure("Unable to decode `data` from BitTorrentPiece")
-      without mhash =? MultiHash.init("sha1", dataBuf).mapFailure, err:
+      without mhash =? MultiHash.init(dataBuf).mapFailure, err:
         return failure(err.msg)
       pieces.add(mhash)
   discard ?pbInfo.getField(4, name).mapFailure

codex/bittorrent/manifest/manifest.nim

@@ -21,6 +21,14 @@ type
     info*: BitTorrentInfo
     codexManifestCid*: Cid
 
+proc `$`*(self: BitTorrentInfo): string =
+  "BitTorrentInfo(length: " & $self.length & ", pieceLength: " & $self.pieceLength &
+    ", pieces: " & $self.pieces & ", name: " & $self.name & ")"
+
+proc `$`*(self: BitTorrentManifest): string =
+  "BitTorrentManifest(info: " & $self.info & ", codexManifestCid: " &
+    $self.codexManifestCid & ")"
+
 proc newBitTorrentManifest*(
     info: BitTorrentInfo, codexManifestCid: Cid
 ): BitTorrentManifest =

codex/node.nim

@@ -159,32 +159,34 @@ proc fetchManifest*(self: CodexNodeRef, cid: Cid): Future[?!Manifest] {.async.}
   manifest.success
 
 proc fetchTorrentManifest*(
-    self: CodexNodeRef, cid: Cid
+    self: CodexNodeRef, infoHashCid: Cid
 ): Future[?!BitTorrentManifest] {.async.} =
-  if err =? cid.isTorrentInfoHash.errorOption:
+  if err =? infoHashCid.isTorrentInfoHash.errorOption:
     return failure "CID has invalid content type for torrent info hash {$cid}"
 
-  trace "Retrieving torrent manifest for cid", cid
+  trace "Retrieving torrent manifest for infoHashCid", infoHashCid
 
-  without blk =? await self.networkStore.getBlock(BlockAddress.init(cid)), err:
-    trace "Error retrieve manifest block", cid, err = err.msg
+  without blk =? await self.networkStore.getBlock(BlockAddress.init(infoHashCid)), err:
+    trace "Error retrieve manifest block", infoHashCid, err = err.msg
     return failure err
 
-  trace "Decoding torrent manifest for cid", cid
+  trace "Successfully retrieved torrent manifest with given block cid",
+    cid = blk.cid, infoHashCid
+  trace "Decoding torrent manifest"
 
   without torrentManifest =? BitTorrentManifest.decode(blk), err:
     trace "Unable to decode torrent manifest", err = err.msg
     return failure("Unable to decode torrent manifest")
 
-  trace "Decoded torrent manifest", cid
+  trace "Decoded torrent manifest", infoHashCid, torrentManifest = $torrentManifest
 
-  without isValid =? torrentManifest.validate(cid), err:
-    trace "Error validating torrent manifest", cid, err = err.msg
+  without isValid =? torrentManifest.validate(infoHashCid), err:
+    trace "Error validating torrent manifest", infoHashCid, err = err.msg
     return failure(err.msg)
 
   if not isValid:
-    trace "Torrent manifest does not match torrent info hash", cid
-    return failure "Torrent manifest does not match torrent info hash {$cid}"
+    trace "Torrent manifest does not match torrent info hash", infoHashCid
+    return failure "Torrent manifest does not match torrent info hash {$infoHashCid}"
 
   return torrentManifest.success
 
@@ -431,6 +433,7 @@ proc streamTorrent(
       error "Piece verification failed", pieceIndex = pieceIndex
       return failure("Piece verification failed")
 
+    trace "Piece verified", pieceIndex, pieceHash
     # great success
     success()
 
@@ -629,12 +632,12 @@ proc storePieces*(
   ## Save stream contents as dataset with given blockSize
   ## to nodes's BlockStore, and return Cid of its manifest
   ##
-  info "Storing data"
+  info "Storing pieces"
 
   let
     hcodec = Sha256HashCodec
     dataCodec = BlockCodec
-    chunker = LPStreamChunker.new(stream, chunkSize = blockSize)
+    chunker = LPStreamChunker.new(stream, chunkSize = blockSize, pad = false)
     numOfBlocksPerPiece = pieceLength.int div blockSize.int
 
   var
@@ -645,13 +648,18 @@ proc storePieces*(
 
   pieceHashCtx.init()
 
+  trace "number of blocks per piece: ", numOfBlocksPerPiece
+
   try:
     while (let chunk = await chunker.getBytes(); chunk.len > 0):
+      trace "storing block...", chunkLength = chunk.len
       if pieceIter.finished:
+        trace "finishing piece..."
         without mh =? MultiHash.init($Sha1HashCodec, pieceHashCtx.finish()).mapFailure,
           err:
           return failure(err)
         pieces.add(mh)
+        trace "successfully computed piece multihash", pieces = $pieces
         pieceIter = Iter[int].new(0 ..< numOfBlocksPerPiece)
         pieceHashCtx.init()
       without mhash =? MultiHash.digest($hcodec, chunk).mapFailure, err:
@@ -669,7 +677,11 @@ proc storePieces*(
         error "Unable to store block", cid = blk.cid, err = err.msg
         return failure(&"Unable to store block {blk.cid}")
       pieceHashCtx.update(chunk)
-      discard pieceIter.next()
+      let idx = pieceIter.next()
+      trace "stored block in piece with index=", idx
+      if chunk.len < blockSize.int:
+        trace "no more block to read"
+        break
   except CancelledError as exc:
     raise exc
   except CatchableError as exc:
@@ -681,6 +693,8 @@ proc storePieces*(
     return failure(err)
   pieces.add(mh)
 
+  trace "finished processing blocks", pieces = $pieces
+
   without tree =? CodexTree.init(cids), err:
     return failure(err)
 
@@ -744,11 +758,17 @@ proc storeTorrent*(
     ):
     return failure("Unable to store BitTorrent data")
 
+  trace "Created BitTorrent manifest", bitTorrentManifest = $bitTorrentManifest
+
   let infoBencoded = bencode(bitTorrentManifest.info)
 
+  trace "BitTorrent Info successfully bencoded"
+
   without infoHash =? MultiHash.digest($Sha1HashCodec, infoBencoded).mapFailure, err:
     return failure(err)
 
+  trace "computed info hash", infoHash = $infoHash
+
   without manifestBlk =? await self.storeBitTorrentManifest(
     bitTorrentManifest, infoHash
   ), err: