Run VMB benchmarks in CI

bitjson · bitjson · commit c0eadacbbc4a · 2024-05-30T16:19:19.000-04:00
diff --git a/.changeset/nervous-carrots-punch.md b/.changeset/nervous-carrots-punch.md
@@ -0,0 +1,5 @@
+---
+'@bitauth/libauth': patch
+---
+
+Run VMB benchmarks in CI
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -26,15 +26,32 @@ jobs:
           CI_NODE_INDEX: ${{ matrix.runner_index }}
           CI_NODE_TOTAL: ${{ matrix.runners_per_version }}
       - run: yarn cov:lcov
-        if: ${{ matrix.node-version }} == 20
+        if: ${{ matrix.node-version == 20 }}
       - name: Upload test coverage
-        if: ${{ matrix.node-version }} == 20
+        if: ${{ matrix.node-version == 20 }}
         uses: codecov/codecov-action@v4
         with:
           fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
           verbose: true
 
+  vmb-benchmarks:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    name: Run VMB Benchmarks
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - name: Install Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+      - run: yarn install --immutable --immutable-cache
+      - run: yarn build
+      - run: yarn bench:vmb_tests
+
   check-policies:
     runs-on: ubuntu-latest
     name: Check Policies
diff --git a/config/.ava.vmb_bench.config.js b/config/.ava.vmb_bench.config.js
@@ -0,0 +1,9 @@
+export default {
+  files: ['src/lib/vmb-tests/benchmark-bch-vmb-tests.spec.ts'],
+  typescript: {
+    compile: false,
+    rewritePaths: {
+      'src/': 'build/',
+    },
+  },
+};
diff --git a/package.json b/package.json
@@ -71,7 +71,7 @@
     "test:unit:vmb_tests": "c8 ava src/lib/vmb-tests/bch-vmb-tests.spec.ts --serial",
     "test:unit:vmb_test": "node --enable-source-maps 'build/lib/vmb-tests/run-bch-vmb-test.spec.helper.js'",
     "dev:vmb_tests": "yarn gen:vmb_tests && yarn build:tsc && ava src/lib/vmb-tests/bch-vmb-tests.spec.ts --serial --fail-fast",
-    "bench:vmb_tests": "ava src/lib/vmb-tests/benchmark-bch-vmb-tests.spec.ts --serial",
+    "bench:vmb_tests": "ava --config config/.ava.vmb_bench.config.js src/lib/vmb-tests/benchmark-bch-vmb-tests.spec.ts --serial",
     "bench": "yarn build && yarn bench:browser-deps && yarn bench:vmb_tests && yarn bench:test",
     "bench:test": "ava --config config/.ava.bench.config.js --serial --timeout=2m 2>&1 | tee bench.log",
     "bench:browser-deps": "cpy '.yarn/artifacts/*.js' build/bench",
@@ -167,7 +167,8 @@
       "--experimental-json-modules",
       "--experimental-global-webcrypto",
       "# ^ needed for node v18"
-    ]
+    ],
+    "files": ["!src/lib/vmb-tests/benchmark-bch-vmb-tests.spec.ts"]
   },
   "config": {
     "commitizen": {
diff --git a/src/lib/address/base58-address.spec.ts b/src/lib/address/base58-address.spec.ts
@@ -15,9 +15,9 @@ import {
 } from '../lib.js';
 
 // eslint-disable-next-line import/no-restricted-paths, import/no-internal-modules
-import keyIoInvalid from './fixtures/key_io_invalid.json' assert { type: 'json' };
+import keyIoInvalid from './fixtures/key_io_invalid.json' with { type: 'json' };
 // eslint-disable-next-line import/no-restricted-paths, import/no-internal-modules
-import keyIoValid from './fixtures/key_io_valid.json' assert { type: 'json' };
+import keyIoValid from './fixtures/key_io_valid.json' with { type: 'json' };
 
 import { fc, testProp } from '@fast-check/ava';
 
diff --git a/src/lib/address/cash-address.spec.ts b/src/lib/address/cash-address.spec.ts
@@ -31,7 +31,7 @@ import {
 } from '../lib.js';
 
 // eslint-disable-next-line import/no-restricted-paths, import/no-internal-modules
-import cashAddrJson from './fixtures/cashaddr.json' assert { type: 'json' };
+import cashAddrJson from './fixtures/cashaddr.json' with { type: 'json' };
 
 import fc from 'fast-check';
 
diff --git a/src/lib/format/base-convert.spec.ts b/src/lib/format/base-convert.spec.ts
@@ -13,7 +13,7 @@ import {
 } from '../lib.js';
 
 // eslint-disable-next-line import/no-restricted-paths, import/no-internal-modules
-import base58Json from './fixtures/base58_encode_decode.json' assert { type: 'json' };
+import base58Json from './fixtures/base58_encode_decode.json' with { type: 'json' };
 
 import { fc, testProp } from '@fast-check/ava';
 
diff --git a/src/lib/key/bip39.spec.ts b/src/lib/key/bip39.spec.ts
@@ -35,9 +35,9 @@ import {
 } from '../lib.js';
 
 // eslint-disable-next-line import/no-restricted-paths, import/no-internal-modules
-import bip39ExtendedVectors from './fixtures/bip39.extended-vectors.json' assert { type: 'json' };
+import bip39ExtendedVectors from './fixtures/bip39.extended-vectors.json' with { type: 'json' };
 // eslint-disable-next-line import/no-restricted-paths, import/no-internal-modules
-import bip39TrezorVectorsRaw from './fixtures/bip39.trezor.json' assert { type: 'json' };
+import bip39TrezorVectorsRaw from './fixtures/bip39.trezor.json' with { type: 'json' };
 
 import { fc, testProp } from '@fast-check/ava';
 import { entropyToMnemonic, mnemonicToSeedSync } from 'bip39';
diff --git a/src/lib/message/transaction-encoding.spec.ts b/src/lib/message/transaction-encoding.spec.ts
@@ -24,9 +24,9 @@ import {
 } from '../lib.js';
 
 // eslint-disable-next-line import/no-restricted-paths, import/no-internal-modules
-import tokenPrefixInvalidJson from './fixtures/token-prefix-invalid.json' assert { type: 'json' };
+import tokenPrefixInvalidJson from './fixtures/token-prefix-invalid.json' with { type: 'json' };
 // eslint-disable-next-line import/no-restricted-paths, import/no-internal-modules
-import tokenPrefixValidJson from './fixtures/token-prefix-valid.json' assert { type: 'json' };
+import tokenPrefixValidJson from './fixtures/token-prefix-valid.json' with { type: 'json' };
 
 test('decodeTransaction', (t) => {
   /**
diff --git a/src/lib/transaction/transaction-e2e.spec.helper.ts b/src/lib/transaction/transaction-e2e.spec.helper.ts
@@ -1,9 +1,9 @@
 /* eslint-disable import/no-internal-modules */
-import twoOfTwoRecoverableJson from './fixtures/templates/2-of-2-recoverable.json' assert { type: 'json' };
-import twoOfThreeJson from './fixtures/templates/2-of-3.json' assert { type: 'json' };
-import cashChannelsJson from './fixtures/templates/cash-channels-v1.json' assert { type: 'json' };
-import p2pkhJson from './fixtures/templates/p2pkh.json' assert { type: 'json' };
-import sigOfSigJson from './fixtures/templates/sig-of-sig.json' assert { type: 'json' };
+import twoOfTwoRecoverableJson from './fixtures/templates/2-of-2-recoverable.json' with { type: 'json' };
+import twoOfThreeJson from './fixtures/templates/2-of-3.json' with { type: 'json' };
+import cashChannelsJson from './fixtures/templates/cash-channels-v1.json' with { type: 'json' };
+import p2pkhJson from './fixtures/templates/p2pkh.json' with { type: 'json' };
+import sigOfSigJson from './fixtures/templates/sig-of-sig.json' with { type: 'json' };
 
 export {
   twoOfTwoRecoverableJson,
diff --git a/src/lib/vm/instruction-sets/common/signing-serialization.spec.ts b/src/lib/vm/instruction-sets/common/signing-serialization.spec.ts
@@ -11,7 +11,7 @@ import {
   numberToBinInt32TwosCompliment,
 } from '../../../lib.js';
 // eslint-disable-next-line import/no-internal-modules, import/no-restricted-paths
-import sighashTests from '../xec/fixtures/satoshi-client/sighash.json' assert { type: 'json' };
+import sighashTests from '../xec/fixtures/satoshi-client/sighash.json' with { type: 'json' };
 
 const tests = Object.values(sighashTests)
   .filter((e) => e.length !== 1 && e.length < 8)
diff --git a/src/lib/vm/instruction-sets/xec/xec.script-tests.spec.ts b/src/lib/vm/instruction-sets/xec/xec.script-tests.spec.ts
@@ -14,9 +14,9 @@ import {
 } from '../../../lib.js';
 
 // eslint-disable-next-line import/no-restricted-paths, import/no-internal-modules
-import scriptTestsAddendum from './fixtures/satoshi-client/script-tests-addendum.json' assert { type: 'json' };
+import scriptTestsAddendum from './fixtures/satoshi-client/script-tests-addendum.json' with { type: 'json' };
 // eslint-disable-next-line import/no-restricted-paths, import/no-internal-modules
-import scriptTests from './fixtures/satoshi-client/script_tests.json' assert { type: 'json' };
+import scriptTests from './fixtures/satoshi-client/script_tests.json' with { type: 'json' };
 
 const tests = Object.values(scriptTests)
   .filter((e) => e.length !== 1 && e.length < 7)
diff --git a/src/lib/vmb-tests/bch-vmb-tests.spec.ts b/src/lib/vmb-tests/bch-vmb-tests.spec.ts
@@ -25,19 +25,19 @@ import {
 import { vmbTestsBch } from './bch-vmb-tests.js';
 import type { TestedVM, VmName } from './bch-vmb-tests.spec.helper.js';
 /* eslint-disable import/no-restricted-paths, import/no-internal-modules */
-import vmbTestsBchChipLoopsInvalidJson from './generated/CHIPs/bch_vmb_tests_chip_loops_invalid.json' assert { type: 'json' };
-import vmbTestsBchChipLoopsNonstandardJson from './generated/CHIPs/bch_vmb_tests_chip_loops_nonstandard.json' assert { type: 'json' };
-import vmbTestsBchChipLoopsStandardJson from './generated/CHIPs/bch_vmb_tests_chip_loops_standard.json' assert { type: 'json' };
-import vmbTestsBchJson from './generated/bch_vmb_tests.json' assert { type: 'json' };
-import vmbTestsBch2023InvalidJson from './generated/bch_vmb_tests_2023_invalid.json' assert { type: 'json' };
-import vmbTestsBch2023NonstandardJson from './generated/bch_vmb_tests_2023_nonstandard.json' assert { type: 'json' };
-import vmbTestsBch2023StandardJson from './generated/bch_vmb_tests_2023_standard.json' assert { type: 'json' };
-import vmbTestsBch2025InvalidJson from './generated/bch_vmb_tests_2025_invalid.json' assert { type: 'json' };
-import vmbTestsBch2025NonstandardJson from './generated/bch_vmb_tests_2025_nonstandard.json' assert { type: 'json' };
-import vmbTestsBch2025StandardJson from './generated/bch_vmb_tests_2025_standard.json' assert { type: 'json' };
-import vmbTestsBch2026InvalidJson from './generated/bch_vmb_tests_2026_invalid.json' assert { type: 'json' };
-import vmbTestsBch2026NonstandardJson from './generated/bch_vmb_tests_2026_nonstandard.json' assert { type: 'json' };
-import vmbTestsBch2026StandardJson from './generated/bch_vmb_tests_2026_standard.json' assert { type: 'json' };
+import vmbTestsBchChipLoopsInvalidJson from './generated/CHIPs/bch_vmb_tests_chip_loops_invalid.json' with { type: 'json' };
+import vmbTestsBchChipLoopsNonstandardJson from './generated/CHIPs/bch_vmb_tests_chip_loops_nonstandard.json' with { type: 'json' };
+import vmbTestsBchChipLoopsStandardJson from './generated/CHIPs/bch_vmb_tests_chip_loops_standard.json' with { type: 'json' };
+import vmbTestsBchJson from './generated/bch_vmb_tests.json' with { type: 'json' };
+import vmbTestsBch2023InvalidJson from './generated/bch_vmb_tests_2023_invalid.json' with { type: 'json' };
+import vmbTestsBch2023NonstandardJson from './generated/bch_vmb_tests_2023_nonstandard.json' with { type: 'json' };
+import vmbTestsBch2023StandardJson from './generated/bch_vmb_tests_2023_standard.json' with { type: 'json' };
+import vmbTestsBch2025InvalidJson from './generated/bch_vmb_tests_2025_invalid.json' with { type: 'json' };
+import vmbTestsBch2025NonstandardJson from './generated/bch_vmb_tests_2025_nonstandard.json' with { type: 'json' };
+import vmbTestsBch2025StandardJson from './generated/bch_vmb_tests_2025_standard.json' with { type: 'json' };
+import vmbTestsBch2026InvalidJson from './generated/bch_vmb_tests_2026_invalid.json' with { type: 'json' };
+import vmbTestsBch2026NonstandardJson from './generated/bch_vmb_tests_2026_nonstandard.json' with { type: 'json' };
+import vmbTestsBch2026StandardJson from './generated/bch_vmb_tests_2026_standard.json' with { type: 'json' };
 
 /* eslint-enable import/no-restricted-paths, import/no-internal-modules */
 
@@ -57,7 +57,7 @@ test('bch_vmb_tests.json is up to date and contains no test ID collisions', asyn
   t.deepEqual(
     allTestCases,
     vmbTestsBchJson,
-    'New test definitions were added to `bch-vmb.tests.ts`, but the generated tests were not updated. Run "yarn gen:vmb-tests" to correct this issue. (Note: tsc watch tasks don\'t always update cached JSON imports when the source file changes. You may need to restart tsc to clear this error after re-generating tests.)',
+    'New test definitions were added to `bch-vmb.tests.ts`, but the generated tests were not updated. Run "yarn gen:vmb_tests" to correct this issue. (Note: tsc watch tasks don\'t always update cached JSON imports when the source file changes. You may need to restart tsc to clear this error after re-generating tests.)',
   );
 
   const testCaseIds = allTestCases.map((testCase) => testCase[0]);
diff --git a/src/lib/vmb-tests/benchmark-bch-vmb-tests.spec.ts b/src/lib/vmb-tests/benchmark-bch-vmb-tests.spec.ts
@@ -16,7 +16,7 @@ import {
   vms,
 } from './bch-vmb-tests.spec.helper.js';
 /* eslint-disable import/no-restricted-paths, import/no-internal-modules */
-import vmbTestsBchJson from './generated/bch_vmb_tests.json' assert { type: 'json' };
+import vmbTestsBchJson from './generated/bch_vmb_tests.json' with { type: 'json' };
 /* eslint-enable import/no-restricted-paths, import/no-internal-modules */
 
 import { Bench } from 'tinybench';
@@ -46,8 +46,8 @@ test('Baseline benchmark has expected VMB test ID', (t) => {
 });
 
 test('Run VMB tests marked with "[benchmark]"', async (t) => {
-  const twentyMinutes = 1_200_000;
-  t.timeout(twentyMinutes);
+  const sixtyMinutes = 3_600_000;
+  t.timeout(sixtyMinutes);
   const plannedBenchmarks = benchmarks.map((testDefinition) => {
     const [
       shortId,
diff --git a/src/lib/vmb-tests/readme.md b/src/lib/vmb-tests/readme.md
@@ -12,12 +12,13 @@ Like the C++ implementation's [`script_tests.json`](../vm/instruction-sets/xec/f
 
 Each VMB test is an array including:
 
-- A short, unique identifier for the test (based on the hash of the test contents)
-- A string describing the purpose/behavior of the test
-- The unlocking script under test (disassembled, i.e. human-readable)
-- The locking script under test (disassembled)
-- The full, encoded test transaction
-- An encoded list of unspent transaction outputs (UTXOs) with which to verify the test transaction (ordered to match the input order of the test transaction)
+0. **`shortId`** - A short, unique identifier for the test (based on the hash of the test contents)
+1. **`testDescription`** - A string describing the purpose/behavior of the test
+2. **`unlockingScriptAsm`** - The unlocking script under test (disassembled, i.e. human-readable)
+3. **`redeemOrLockingScriptAsm`** - The locking script under test (disassembled)
+4. **`testTransactionHex`** - The full, encoded test transaction
+5. **`sourceOutputsHex`** - An encoded list of unspent transaction outputs (UTXOs) with which to verify the test transaction (ordered to match the input order of the test transaction)
+6. **`inputIndex` (default: `0`)** - The input index of the primary input under test (evaluating the scripts from array index `2` and `3` above); if not specified, the primary input under test is input `0`.
 
 Every test vector in each VM's master test file (e.g. [`bch_vmb_tests.json`](./generated/bch_vmb_tests.json)) also includes a list of labels indicating the VM configurations for which the test vector applies. This master test file is automatically broken up into a variety of smaller, single-configuration test files for easier use (e.g. [`bch_vmb_tests_2025_standard.json`](./generated/bch_vmb_tests_2025_standard.json)).
 
@@ -53,4 +54,4 @@ For an example of VMB test usage, see [`bch-vmb-tests.spec.ts`](./bch-vmb-tests.
 
 ### Generating VMB Tests
 
-Libauth's VMB tests are generated by the `yarn gen:vmb-tests` package script and committed to the repo. (Libauth's continuous integration tests also ensure that VMB tests remain up to date and passing on Libauth's VM implementations.)
+Libauth's VMB tests are generated by the `yarn gen:vmb_tests` package script and committed to the repo. (Libauth's continuous integration tests also ensure that VMB tests remain up to date and passing on Libauth's VM implementations.)
diff --git a/src/lib/vmb-tests/run-bch-vmb-test.spec.helper.ts b/src/lib/vmb-tests/run-bch-vmb-test.spec.helper.ts
@@ -12,7 +12,7 @@ import {
 
 import { baselineBenchmarkId, isVm, vms } from './bch-vmb-tests.spec.helper.js';
 // eslint-disable-next-line import/no-internal-modules
-import vmbTestsBchJson from './generated/bch_vmb_tests.json' assert { type: 'json' };
+import vmbTestsBchJson from './generated/bch_vmb_tests.json' with { type: 'json' };
 
 import { Bench } from 'tinybench';
 
diff --git a/src/lib/vmb-tests/write-bch-vmb-tests.spec.helper.ts b/src/lib/vmb-tests/write-bch-vmb-tests.spec.helper.ts
@@ -1,7 +1,7 @@
 /* eslint-disable functional/no-expression-statements */
 
 /**
- * This script generates all bch_vmb_tests, run it with: `yarn gen:vmb-tests`.
+ * This script generates all bch_vmb_tests, run it with: `yarn gen:vmb_tests`.
  */
 import { writeFileSync } from 'node:fs';
 import { resolve } from 'node:path';
diff --git a/src/lib/vmb-tests/write-reasons-bch-vmb-tests.spec.helper.ts b/src/lib/vmb-tests/write-reasons-bch-vmb-tests.spec.helper.ts
@@ -1,7 +1,7 @@
 /* eslint-disable functional/no-expression-statements */
 /**
  * This script produces a `*reasons.json` file for every VMB test that is
- * expected to fail. Run it with: `yarn gen:vmb-tests`.
+ * expected to fail. Run it with: `yarn gen:vmb_tests`.
  */
 import { readFileSync, writeFileSync } from 'node:fs';
 import { resolve } from 'node:path';

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +'@bitauth/libauth': patch
 +---
++
 +Run VMB benchmarks in CI