feat: a few improvements to the dry-run, allocations and snapshot update scripts

billing/functions/usage-table.js

@@ -28,7 +28,9 @@ export const handler = Sentry.AWSLambda.wrapHandler(
     if (!stripeSecretKey) throw new Error('missing secret: STRIPE_SECRET_KEY')
 
     const records = parseUsageInsertEvent(event)
-    if (!records.length) return
+    if (!records.length) {
+      throw new Error(`found no records in usage insert event: ${JSON.stringify(event)}`)
+    }
 
     if (records.length > 1) {
       throw new Error(`invalid batch size, expected: 1, actual: ${records.length}`)
@@ -52,6 +54,7 @@ export const handler = Sentry.AWSLambda.wrapHandler(
 const parseUsageInsertEvent = event => {
   const records = []
   for (const r of event.Records) {
+    console.log(`processing usage record: ${JSON.stringify(r)}`)
     if (r.eventName !== 'INSERT') continue
     if (!r.dynamodb) continue
     if (!r.dynamodb.NewImage) throw new Error('missing "NEW_IMAGE" in stream event')

billing/lib/space-billing-queue.js

@@ -63,7 +63,7 @@ export const calculatePeriodUsage = async (instruction, ctx) => {
   let size = snap?.size ?? 0n
   let usage = size * BigInt(instruction.to.getTime() - instruction.from.getTime())
 
-  console.log(`Total size is ${size} bytes @ ${instruction.from.toISOString()}`)
+  console.log(`Total size of ${instruction.space} is ${size} bytes @ ${instruction.from.toISOString()}`)
 
   for await (const page of iterateSpaceDiffs(instruction, ctx)) {
     if (page.error) return page
@@ -74,7 +74,7 @@ export const calculatePeriodUsage = async (instruction, ctx) => {
     }
   }
 
-  console.log(`Total size is ${size} bytes @ ${instruction.to.toISOString()}`)
+  console.log(`Total size of ${instruction.space} is ${size} bytes @ ${instruction.to.toISOString()}`)
 
   return { ok: { size, usage } }
 }
@@ -106,7 +106,7 @@ export const storeSpaceUsage = async (instruction, { size, usage }, ctx) => {
   if (snapPut.error) return snapPut
 
   const duration = instruction.to.getTime() - instruction.from.getTime()
-  console.log(`Space consumed ${usage} byte/ms (~${new Big(usage.toString()).div(duration).div(GB).toFixed(2)} GiB/month)`)
+  console.log(`Space consumed by ${instruction.space} is ${usage} byte/ms (~${new Big(usage.toString()).div(duration).div(GB).toFixed(2)} GiB/month)`)
   const usagePut = await ctx.usageStore.put({
     ...instruction,
     usage,

billing/scripts/dry-run/.env.template

@@ -1,7 +1,2 @@
 AWS_REGION=
-
-CUSTOMER_TABLE_NAME=
-SUBSCRIPTION_TABLE_NAME=
-CONSUMER_TABLE_NAME=
-SPACE_DIFF_TABLE_NAME=
-SPACE_SNAPSHOT_TABLE_NAME=
+STORACHA_ENV=

billing/scripts/dry-run/dry-run.js

@@ -34,11 +34,12 @@ dotenv.config({ path: '.env.local' })
 
 const concurrency = 5
 
-const CUSTOMER_TABLE_NAME = mustGetEnv('CUSTOMER_TABLE_NAME')
-const SUBSCRIPTION_TABLE_NAME = mustGetEnv('SUBSCRIPTION_TABLE_NAME')
-const CONSUMER_TABLE_NAME = mustGetEnv('CONSUMER_TABLE_NAME')
-const SPACE_DIFF_TABLE_NAME = mustGetEnv('SPACE_DIFF_TABLE_NAME')
-const SPACE_SNAPSHOT_TABLE_NAME = mustGetEnv('SPACE_SNAPSHOT_TABLE_NAME')
+const STORACHA_ENV = mustGetEnv('STORACHA_ENV')
+const CUSTOMER_TABLE_NAME=`${STORACHA_ENV}-w3infra-customer`
+const SUBSCRIPTION_TABLE_NAME=`${STORACHA_ENV}-w3infra-subscription`
+const CONSUMER_TABLE_NAME=`${STORACHA_ENV}-w3infra-consumer`
+const SPACE_DIFF_TABLE_NAME = `${STORACHA_ENV}-w3infra-space-diff`
+const SPACE_SNAPSHOT_TABLE_NAME = `${STORACHA_ENV}-w3infra-space-snapshot`
 
 const dynamo = new DynamoDBClient()
 

billing/scripts/space-allocations-snapshot/.env.template

@@ -1,8 +1,2 @@
-AWS_REGION=us-west-2
-
-CUSTOMER_TABLE_NAME=envName-w3infra-customer
-SUBSCRIPTION_TABLE_NAME=envName-w3infra-subscription
-CONSUMER_TABLE_NAME=envName-w3infra-consumer
-SPACE_SNAPSHOT_TABLE_NAME=envName-w3infra-space-snapshot
-ALLOCATIONS_TABLE_NAME=envName-w3infra-allocation
-STORE_TABLE_NAME=envName-w3infra-store
+AWS_REGION=
+STORACHA_ENV=

billing/scripts/space-allocations-snapshot/index.js

@@ -44,11 +44,13 @@ dotenv.config({ path: '.env.local' })
 const concurrency = 5
 const dynamo = new DynamoDBClient()
 
-const CUSTOMER_TABLE_NAME = mustGetEnv('CUSTOMER_TABLE_NAME')
-const SUBSCRIPTION_TABLE_NAME = mustGetEnv('SUBSCRIPTION_TABLE_NAME')
-const CONSUMER_TABLE_NAME = mustGetEnv('CONSUMER_TABLE_NAME')
-const ALLOCATIONS_TABLE_NAME = mustGetEnv('ALLOCATIONS_TABLE_NAME')
-const STORE_TABLE_NAME = mustGetEnv('STORE_TABLE_NAME')
+const STORACHA_ENV = mustGetEnv('STORACHA_ENV')
+
+const CUSTOMER_TABLE_NAME=`${STORACHA_ENV}-w3infra-customer`
+const SUBSCRIPTION_TABLE_NAME=`${STORACHA_ENV}-w3infra-subscription`
+const CONSUMER_TABLE_NAME=`${STORACHA_ENV}-w3infra-consumer`
+const ALLOCATIONS_TABLE_NAME=`${STORACHA_ENV}-w3infra-allocation`
+const STORE_TABLE_NAME=`${STORACHA_ENV}-w3infra-store`
 
 const customerStore = createCustomerStore(dynamo, {
   tableName: CUSTOMER_TABLE_NAME,

billing/scripts/space-allocations-snapshot/upsert-snapshot-table.js

@@ -19,16 +19,21 @@ import { mustGetEnv } from '../../../lib/env.js'
 
 dotenv.config({ path: '.env.local' })
 
-const SPACE_SNAPSHOT_TABLE_NAME = mustGetEnv('SPACE_SNAPSHOT_TABLE_NAME')
+const STORACHA_ENV = mustGetEnv('STORACHA_ENV')
+const SPACE_SNAPSHOT_TABLE_NAME = `${STORACHA_ENV}-w3infra-space-snapshot`
 
 const args = process.argv.slice(2)
 const filename = args[0] // TODO: validate
 
 const dynamo = new DynamoDBClient()
 
-export async function main() {
-  /** @type ParsedSnapshot[]} */
-  const snapshots = await readCsvInput(filename)
+/**
+ * Write up to 100 snapshots
+ * 
+ * @param {ParsedSnapshot[]} snapshots 
+ */
+async function upsertSnapshots(snapshots) {
+  if (snapshots.length > 100) throw new Error('cannot write more than 100 snapshots in one batch - https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/dynamodb/command/TransactWriteItemsCommand/')
 
   /** @type {import('@aws-sdk/client-dynamodb').TransactWriteItem[]} */
   const transactItems = []
@@ -52,12 +57,25 @@ export async function main() {
   /** @type {import('@aws-sdk/client-dynamodb').TransactWriteItemsCommandInput} */
   const transactWriteParams = { TransactItems: transactItems }
 
-  try {
-    const command = new TransactWriteItemsCommand(transactWriteParams)
-    const response = await dynamo.send(command)
-    console.log(response)
-  } catch (err) {
-    console.error('Failed to update or create snapshot!', err)
+  const command = new TransactWriteItemsCommand(transactWriteParams)
+  return await dynamo.send(command)
+}
+
+export async function main() {
+  /** @type ParsedSnapshot[]} */
+  const snapshots = await readCsvInput(filename)
+  console.log(`writing ${snapshots.length} snapshots to ${STORACHA_ENV}'s ${SPACE_SNAPSHOT_TABLE_NAME} dynamo table`)
+  for (let i = 0; i < snapshots.length; i += 100) {
+    const batch = [];
+    for (let j = i; j < i + 100 && j < snapshots.length; j++) {
+      batch.push(snapshots[j]);
+    }
+    try {
+      const response = await upsertSnapshots(batch)
+      console.log(response)
+    } catch (err) {
+      console.error(`Failed to update or create batch ${JSON.stringify(batch, null, 4)}`, err)
+    }
   }
 }