From d49918e394f3f588712270ac57af16d129f77786 Mon Sep 17 00:00:00 2001 From: Stephan Cilliers <5469870+stephancill@users.noreply.github.com> Date: Thu, 19 Sep 2024 21:51:47 +0200 Subject: [PATCH] fix: db migration issue --- docker-compose.yml | 56 ++++++++++++++++++++--------------------- src/api/cast.ts | 2 +- src/api/fid.ts | 2 +- src/api/hub.ts | 2 +- src/api/link.ts | 2 +- src/api/reaction.ts | 2 +- src/api/signer.ts | 2 +- src/api/storage.ts | 2 +- src/api/user-data.ts | 2 +- src/api/verification.ts | 2 +- src/db/db.ts | 40 +++++++++++++++++++++++++++++ src/db/kysely.ts | 18 ------------- src/db/migrator.ts | 4 ++- src/index.ts | 4 +-- src/lib/links-utils.ts | 3 +++ src/lib/targets.ts | 2 +- 16 files changed, 86 insertions(+), 59 deletions(-) create mode 100644 src/db/db.ts delete mode 100644 src/db/kysely.ts diff --git a/docker-compose.yml b/docker-compose.yml index 4c97bae..935867c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,10 +14,10 @@ services: - postgres-data:/var/lib/postgresql/data healthcheck: test: ['CMD-SHELL', 'pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB'] - interval: 5s # Check every 5 seconds for readiness - timeout: 5s # Allow up to 5 seconds for a response - retries: 3 # Fail after 3 unsuccessful attempts - start_period: 10s # Start checks after 10 seconds + interval: 5s # Check every 5 seconds for readiness + timeout: 5s # Allow up to 5 seconds for a response + retries: 3 # Fail after 3 unsuccessful attempts + start_period: 10s # Start checks after 10 seconds networks: - indexer-network @@ -31,33 +31,33 @@ services: - '6379:6379' healthcheck: test: ['CMD-SHELL', 'redis-cli ping'] - interval: 5s # Check every 5 seconds - timeout: 5s # Allow up to 5 seconds for a response - retries: 3 # Fail after 3 unsuccessful attempts - start_period: 5s # Start health checks after 5 seconds + interval: 5s # Check every 5 seconds + timeout: 5s # Allow up to 5 seconds for a response + retries: 3 # Fail after 3 unsuccessful attempts + start_period: 5s # Start health checks after 5 seconds networks: - indexer-network - lazy-indexer: - build: . - restart: unless-stopped - depends_on: - postgres: - condition: service_healthy - redis: - condition: service_healthy - environment: - - DATABASE_URL=postgresql://indexer:password@postgres:5432/indexer - - REDIS_URL=redis://redis:6379 - - HUB_REST_URL=https://hub.pinata.cloud - - HUB_RPC=hub-grpc.pinata.cloud - - HUB_SSL=true - - WORKER_CONCURRENCY=5 - - LOG_LEVEL=debug - networks: - - indexer-network - command: > - /bin/sh -c "yarn backfill & yarn stream" + # lazy-indexer: + # build: . + # restart: unless-stopped + # depends_on: + # postgres: + # condition: service_healthy + # redis: + # condition: service_healthy + # environment: + # - DATABASE_URL=postgresql://indexer:password@postgres:5432/indexer + # - REDIS_URL=redis://redis:6379 + # - HUB_REST_URL=https://hub.pinata.cloud + # - HUB_RPC=hub-grpc.pinata.cloud + # - HUB_SSL=true + # - WORKER_CONCURRENCY=5 + # - LOG_LEVEL=debug + # networks: + # - indexer-network + # command: > + # /bin/sh -c "yarn backfill & yarn stream" volumes: postgres-data: diff --git a/src/api/cast.ts b/src/api/cast.ts index 436a07f..efd48f7 100644 --- a/src/api/cast.ts +++ b/src/api/cast.ts @@ -4,7 +4,7 @@ import { isCastRemoveMessage, } from '@farcaster/hub-nodejs' -import { db } from '../db/kysely.js' +import { db } from '../db/db.js' import { getBackfillQueue, queueBackfillJob } from '../lib/backfill.js' import { log } from '../lib/logger.js' import { diff --git a/src/api/fid.ts b/src/api/fid.ts index c2123c2..24f6926 100644 --- a/src/api/fid.ts +++ b/src/api/fid.ts @@ -5,7 +5,7 @@ import { isIdRegisterOnChainEvent, } from '@farcaster/hub-nodejs' -import { db } from '../db/kysely.js' +import { db } from '../db/db.js' import { hubClient } from '../lib/hub-client.js' import { getOnChainEventsByFidInBatchesOf } from '../lib/paginate.js' import { MAX_PAGE_SIZE, NULL_ETH_ADDRESS } from '../lib/utils.js' diff --git a/src/api/hub.ts b/src/api/hub.ts index e8f528d..380af95 100644 --- a/src/api/hub.ts +++ b/src/api/hub.ts @@ -1,6 +1,6 @@ import { ContactInfoContentBody } from '@farcaster/hub-nodejs' -import { db } from '../db/kysely.js' +import { db } from '../db/db.js' import { log } from '../lib/logger.js' import { breakIntoChunks, formatHubs } from '../lib/utils.js' diff --git a/src/api/link.ts b/src/api/link.ts index 097c717..ab3c310 100644 --- a/src/api/link.ts +++ b/src/api/link.ts @@ -4,7 +4,7 @@ import { isLinkRemoveMessage, } from '@farcaster/hub-nodejs' -import { db } from '../db/kysely.js' +import { db } from '../db/db.js' import { getBackfillQueue, queueBackfillJob } from '../lib/backfill.js' import { log } from '../lib/logger.js' import { isRootTarget } from '../lib/targets.js' diff --git a/src/api/reaction.ts b/src/api/reaction.ts index d4bef6f..e7a9106 100644 --- a/src/api/reaction.ts +++ b/src/api/reaction.ts @@ -4,7 +4,7 @@ import { isReactionRemoveMessage, } from '@farcaster/hub-nodejs' -import { db } from '../db/kysely.js' +import { db } from '../db/db.js' import { log } from '../lib/logger.js' import { breakIntoChunks, diff --git a/src/api/signer.ts b/src/api/signer.ts index a181183..b5af0e4 100644 --- a/src/api/signer.ts +++ b/src/api/signer.ts @@ -6,7 +6,7 @@ import { } from '@farcaster/hub-nodejs' import { bytesToHex, decodeAbiParameters } from 'viem' -import { db } from '../db/kysely.js' +import { db } from '../db/db.js' import { hubClient } from '../lib/hub-client.js' import { getOnChainEventsByFidInBatchesOf } from '../lib/paginate.js' import { MAX_PAGE_SIZE } from '../lib/utils.js' diff --git a/src/api/storage.ts b/src/api/storage.ts index 4862935..59f615d 100644 --- a/src/api/storage.ts +++ b/src/api/storage.ts @@ -4,7 +4,7 @@ import { isStorageRentOnChainEvent, } from '@farcaster/hub-nodejs' -import { db } from '../db/kysely.js' +import { db } from '../db/db.js' import { hubClient } from '../lib/hub-client.js' import { getOnChainEventsByFidInBatchesOf } from '../lib/paginate.js' import { MAX_PAGE_SIZE, farcasterTimeToDate } from '../lib/utils.js' diff --git a/src/api/user-data.ts b/src/api/user-data.ts index 2c5653a..8cc5328 100644 --- a/src/api/user-data.ts +++ b/src/api/user-data.ts @@ -1,6 +1,6 @@ import { Message } from '@farcaster/hub-nodejs' -import { db } from '../db/kysely.js' +import { db } from '../db/db.js' import { log } from '../lib/logger.js' import { formatUserDatas } from '../lib/utils.js' diff --git a/src/api/verification.ts b/src/api/verification.ts index f3ec14d..ae697de 100644 --- a/src/api/verification.ts +++ b/src/api/verification.ts @@ -4,7 +4,7 @@ import { isVerificationRemoveMessage, } from '@farcaster/hub-nodejs' -import { db } from '../db/kysely.js' +import { db } from '../db/db.js' import { log } from '../lib/logger.js' import { farcasterTimeToDate, formatVerifications } from '../lib/utils.js' diff --git a/src/db/db.ts b/src/db/db.ts new file mode 100644 index 0000000..42dd714 --- /dev/null +++ b/src/db/db.ts @@ -0,0 +1,40 @@ +import 'dotenv/config' +import { CamelCasePlugin, Kysely, PostgresDialect } from 'kysely' +import pg from 'pg' + +import { Tables } from './db.types' + +const { Pool } = pg + +export const getDbClient = ( + connectionString: string | undefined = process.env.DATABASE_URL +) => { + if (!connectionString) { + throw new Error('DATABASE_URL is not set') + } + + return new Kysely({ + dialect: new PostgresDialect({ + pool: new Pool({ + max: 20, + connectionString, + }), + }), + plugins: [new CamelCasePlugin()], + log: ['error'], + }) +} + +export const db = getDbClient() + +// export const db = new Kysely({ +// dialect: new PostgresDialect({ +// pool: new Pool({ +// connectionString: process.env.DATABASE_URL, +// log(...messages) { +// console.log('DB:', ...messages) +// }, +// }), +// }), +// plugins: [new CamelCasePlugin()], +// }) diff --git a/src/db/kysely.ts b/src/db/kysely.ts deleted file mode 100644 index 51078ab..0000000 --- a/src/db/kysely.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { CamelCasePlugin, Kysely, PostgresDialect } from 'kysely' -import Pool from 'pg-pool' - -import { Tables } from './db.types' - -if (!process.env.DATABASE_URL) { - throw new Error('DATABASE_URL is not set') -} - -export const db = new Kysely({ - dialect: new PostgresDialect({ - pool: new Pool({ - max: 10, - connectionString: process.env.DATABASE_URL, - }), - }), - plugins: [new CamelCasePlugin()], -}) diff --git a/src/db/migrator.ts b/src/db/migrator.ts index 2bbcab1..2c7e28e 100644 --- a/src/db/migrator.ts +++ b/src/db/migrator.ts @@ -5,9 +5,11 @@ import * as path from 'path' import { fileURLToPath } from 'url' import { log } from '../lib/logger.js' -import { db } from './kysely.js' +import { getDbClient } from './db.js' export async function migrateToLatest() { + const db = getDbClient() + const migrator = new Migrator({ db, provider: new FileMigrationProvider({ diff --git a/src/index.ts b/src/index.ts index cc95444..ccc7f9d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,12 +1,12 @@ import 'dotenv/config' import { getLatestEvent } from './api/event.js' +import { migrateToLatest } from './db/migrator.js' import { backfill } from './lib/backfill.js' import { initExpressApp } from './lib/express.js' import { subscribe } from './lib/subscriber.js' -import { migrateToLatest } from './db/migrator.js' -migrateToLatest() +await migrateToLatest() if (process.argv[2] === '--backfill') { initExpressApp() diff --git a/src/lib/links-utils.ts b/src/lib/links-utils.ts index aa1a0dd..0679c40 100644 --- a/src/lib/links-utils.ts +++ b/src/lib/links-utils.ts @@ -15,6 +15,7 @@ import fastq from 'fastq' import { createPublicClient, http, parseAbi } from 'viem' import { optimism } from 'viem/chains' +import { log } from './logger.js' import { getAllLinksByFid } from './paginate.js' const HUB_URL = process.env.HUB_REST_URL! @@ -218,6 +219,8 @@ export async function _getAllLinksByFid( }) : await getAllLinksByFid({ fid }) + log.info(`Got ${linksMessages.length} link messages for fid ${fid}`) + const linksSet = new Set() linksMessages.forEach((linkMessage) => diff --git a/src/lib/targets.ts b/src/lib/targets.ts index 902bac2..de8e7f5 100644 --- a/src/lib/targets.ts +++ b/src/lib/targets.ts @@ -1,4 +1,4 @@ -import { db } from '../db/kysely.js' +import { db } from '../db/db.js' import { redis } from './redis.js' export const rootTargetsKey = 'targets:root'