fix: use car block validator (#75)

* fix: use car block validtor

* chore: update car block validator

Author: vasco-santos

package-lock.json

@@ -25,6 +25,7 @@
     "@ipld/car": "4.1.6",
     "@ipld/dag-cbor": "7.0.3",
     "@ipld/dag-pb": "2.1.18",
+    "@web3-storage/car-block-validator": "^1.0.1",
     "dag-jose": "1.0.0",
     "dotenv": "16.0.3",
     "hdr-histogram-js": "3.0.0",
@@ -43,7 +44,8 @@
     "eslint-plugin-promise": "6.1.0",
     "pino-pretty": "9.1.1",
     "prettier": "2.7.1",
-    "tap": "16.3.0"
+    "tap": "16.3.0",
+    "uint8arrays": "3.1.1"
   },
   "engines": {
     "node": "16"

package.json

@@ -1,5 +1,7 @@
 'use strict'
 
+const { validateBlock } = require('@web3-storage/car-block-validator')
+
 const config = require('../config')
 const { queuedTasks } = require('./util')
 const { serializeError } = require('./logging')
@@ -66,6 +68,18 @@ async function writeBlocksBatch({ blocks, car, logger }) {
       block.key = cidToKey(block.cid)
     }
 
+    // Validate block
+    try {
+      await validateBlock({
+        cid: block.cid,
+        bytes: block.data
+      })
+    } catch (error) {
+      logger.error({ error: serializeError(error) }, 'Error validating block')
+      console.log('not valid block', error)
+      continue
+    }
+
     // HOTFIX
     if (keys.includes(block.key)) {
       continue

src/lib/block.js

@@ -20,16 +20,15 @@ class CarIterator {
       const offset = (this.position = this.reader.pos)
       const { cid, length, blockLength } = await readBlockHead(this.reader)
       this.currentCid = cid
-      // disabled decodeBlocks https://github.com/elastic-ipfs/indexer-lambda/pull/54#discussion_r913665164
-      // const data = this.decodeBlocks && cid.code !== RAW_BLOCK_CODEC ? await this.reader.exactly(blockLength) : undefined
+      const data = await this.reader.exactly(blockLength)
 
       yield {
         cid,
         blockLength,
         blockOffset: this.reader.pos,
         offset,
         length,
-        data: undefined
+        data
       }
 
       this.reader.seek(blockLength)

src/lib/iterator.js

@@ -186,6 +186,66 @@ Object {
 }
 `
 
+exports[`test/index.test.js TAP handler indexes a new car file but not invalid blocks > must match snapshot 1`] = `
+Object {
+  "dynamo": Object {
+    "batchCreates": Array [
+      Object {
+        "RequestItems": Object {
+          "v1-blocks-cars-position": Array [],
+        },
+      },
+    ],
+    "creates": Array [
+      Object {
+        "Item": Object {
+          "bucket": Object {
+            "S": "cars",
+          },
+          "bucketRegion": Object {
+            "S": "us-east-2",
+          },
+          "createdAt": Object {
+            "S": "2022-06-24T15:15:17.401Z",
+          },
+          "key": Object {
+            "S": "file3.car",
+          },
+          "path": Object {
+            "S": "us-east-2/cars/file3.car",
+          },
+          "roots": Object {
+            "L": Array [
+              Object {
+                "S": "bafkreiadsbmmn4waznesyuz3bjgrj33xzqhxrk6mz3ksq7meugrachh3qe",
+              },
+            ],
+          },
+          "version": Object {
+            "N": "1",
+          },
+        },
+        "TableName": "v1-cars",
+      },
+    ],
+  },
+  "sqs": Object {
+    "batchPublishes": Array [
+      Object {
+        "Entries": Array [
+          Object {
+            "Id": "zQmNaZbPKs9E11qawNuEs7mSW7KXQ7TQ6Y2X2ovcvE1sUJk",
+            "MessageBody": "zQmNaZbPKs9E11qawNuEs7mSW7KXQ7TQ6Y2X2ovcvE1sUJk",
+          },
+        ],
+        "QueueUrl": "publishingQueue",
+      },
+    ],
+    "publishes": Array [],
+  },
+}
+`
+
 exports[`test/index.test.js TAP handler indexes a new car file with unsupported blocks > must match snapshot 1`] = `
 Object {
   "dynamo": Object {