Deploy feat/token snapshot

.env.defaults

@@ -1,6 +1,7 @@
 SOLANA_RPC=http://solrpc1.aleph.cloud:7725/
-INDEXER=spl-token
-SPL_TOKEN_MINTS=kinXdEcpDQeHPEuQnqmUgtYykqKGVFq6CeVX5iAHJq6
+INDEXER=token-snapshot
+SPL_TOKEN_MINTS=mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So
+SPL_TOKEN_ACCOUNTS=CrR7DS7A8ABSsHwx92K3b6bD1moBzn5SpWf2ske8bqML
 SOLANA_MAIN_PUBLIC_RPC=https://api.mainnet-beta.solana.com
 
 # 16 GB RAM for node.js

.github/workflows/main.yml

@@ -3,18 +3,10 @@ on:
     types: [ labeled ]
 
 jobs:
-  clean:
-    if: ${{ github.event.label.name == 'deploy' }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Cleanup Artifacts
-        uses: glassechidna/artifact-cleaner@master
-
   build:
     if: ${{ github.event.label.name == 'deploy' }}
     runs-on: ubuntu-latest
     name: Build the indexer image
-    needs: clean
     steps:
       - name: Checkout
         uses: actions/checkout@v3

packages/marinade_finance/package.json

@@ -15,8 +15,8 @@
   "author": "ALEPH.im",
   "license": "ISC",
   "dependencies": {
-    "@aleph-indexer/core": "^1.0.23",
-    "@aleph-indexer/framework": "^1.0.23",
+    "@aleph-indexer/core": "1.0.23",
+    "@aleph-indexer/framework": "1.0.23",
     "@metaplex-foundation/beet": "0.7.1",
     "@metaplex-foundation/beet-solana": "0.4.0",
     "@solana/spl-token": "0.3.5",

packages/spl-lending/package.json

@@ -14,10 +14,10 @@
   "author": "ALEPH.im",
   "license": "ISC",
   "dependencies": {
-    "@aleph-indexer/core": "^1.0.23",
-    "@aleph-indexer/framework": "^1.0.23",
+    "@aleph-indexer/core": "1.0.23",
+    "@aleph-indexer/framework": "1.0.23",
     "@aleph-indexer/layout": "^1.0.14",
-    "@aleph-indexer/middleware": "^1.0.23",
+    "@aleph-indexer/middleware": "1.0.23",
     "@port.finance/port-sdk": "^0.2.67",
     "@solendprotocol/solend-sdk": "^0.7.0"
   }

packages/spl-token/package.json

@@ -14,9 +14,9 @@
   "author": "ALEPH.im",
   "license": "ISC",
   "dependencies": {
-    "@aleph-indexer/core": "^1.0.23",
-    "@aleph-indexer/framework": "^1.0.23",
+    "@aleph-indexer/core": "1.0.23",
+    "@aleph-indexer/framework": "1.0.23",
     "@aleph-indexer/layout": "^1.0.14",
-    "@aleph-indexer/middleware": "^1.0.23"
+    "@aleph-indexer/middleware": "1.0.23"
   }
 }

packages/token-snapshot/README.md

@@ -0,0 +1,68 @@
+# SPL Token Snapshot
+
+## Configuration
+
+Set the `SPL_TOKEN_MINTS` environment variable with the token mints
+separated by comas to index these tokens, similar to the [SPL Token](https://github.com/aleph-im/solana-indexer-library/tree/main/packages/spl-token) indexer.
+You can also set the `SPL_TOKEN_ACCOUNTS` environment variable with the specific accounts you want to index.
+
+### Example
+
+To index the mSOL token balances, set the `SPL_TOKEN_MINTS` environment variable to:
+
+``` sh
+SPL_TOKEN_MINTS=mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So
+```
+
+Currently, the [.env.default](https://github.com/MHHukiewitz/solana-indexer-library/blob/feat/token-snapshot/.env.defaults)
+file is configured to index the mSOL token mint with the `token-snapshot` indexer.
+Therefore, you can just run
+
+``` sh
+npm run start token-snapshot
+```
+
+from the root of the project to start the indexer locally. A GraphiQL interface will be available at http://localhost:8080.
+
+It will take a while to index all the accounts, but you can see the progress when querying:
+
+``` graphql
+accountState{
+  account
+  accurate
+  progress
+  pending
+  processed
+}
+```
+
+To get all the balances of accounts with at least 1 lamport of the token on Sun Jan 01 2023 00:00:00 GMT+0000, you can
+query:
+
+``` graphql
+  tokenHolders(
+    gte: "1",
+    mint: "mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So"
+    timestamp: 1672531200000
+  ) {
+    timestamp
+    owner
+    account
+    balances {
+      total
+    }
+  }
+```
+
+### ToDos
+
+- [ ] Add indicator whether an account belongs to a Solana program
+- [ ] Add parsing of lending program instructions to also index the collateral and borrow balances
+  - [ ] Solend
+  - [ ] Port
+  - [ ] Larix
+
+## Deploying the indexer
+
+Follow the guide of the main [README.md](https://github.com/aleph-im/solana-indexer-library/blob/main/README.md) file
+to publish it in Aleph.IM network using different methods.

packages/token-snapshot/docker-compose.yaml

@@ -0,0 +1,18 @@
+version: '2'
+
+services:
+  spl-token-indexer:
+    build: ../..
+    volumes:
+      - ~/indexer.data:/app/data:rw
+    extra_hosts:
+      - host.docker.internal:host-gateway
+    env_file:
+      - ../../.env
+      - ./.env
+    environment:
+      - INDEXER=token-snapshot
+      - SPL_TOKEN_MINTS=mSoLzYCxHdYgdzU16g5QSh3i5K3z3KZK7ytfqcJm7So
+      - SOLANA_RPC=http://solrpc1.aleph.cloud:7725/
+      - INDEXER_INSTANCES=1
+    network_mode: bridge

packages/token-snapshot/index.ts

@@ -0,0 +1 @@
+// todo

packages/token-snapshot/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@aleph-indexer/token-snapshot",
+  "version": "1.0.0",
+  "description": "",
+  "main": "dist/index.js",
+  "module": "dist/index.js",
+  "types": "dist/index.d.js",
+  "type": "module",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "up": "docker-compose up -d",
+    "up:devnet": "docker-compose -f docker-compose-devnet.yaml --project-name staking-devnet up -d"
+  },
+  "author": "ALEPH.im",
+  "license": "ISC",
+  "dependencies": {
+    "@aleph-indexer/core": "1.0.23",
+    "@aleph-indexer/framework": "1.0.23",
+    "@aleph-indexer/layout": "^1.0.14",
+    "@aleph-indexer/middleware": "1.0.23"
+  }
+}

packages/token-snapshot/run.ts

@@ -0,0 +1,52 @@
+import { fileURLToPath } from 'url'
+import path from 'path'
+import { config } from '@aleph-indexer/core'
+import SDK, { TransportType } from '@aleph-indexer/framework'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = path.dirname(__filename)
+
+async function main() {
+  const workerDomainPath = path.join(__dirname, './src/domain/worker.js')
+  const mainDomainPath = path.join(__dirname, './src/domain/main.js')
+  const apiSchemaPath = path.join(__dirname, './src/api/index.js')
+
+  const instances = Number(config.INDEXER_INSTANCES || 1)
+  const apiPort = Number(config.INDEXER_API_PORT || 8080)
+  const tcpUrls = config.INDEXER_TCP_URLS || undefined
+  const natsUrl = config.INDEXER_NATS_URL || undefined
+
+  const projectId = config.INDEXER_NAMESPACE || 'token-snapshot'
+  const dataPath = config.INDEXER_DATA_PATH || undefined // 'data'
+  const transport =
+    (config.INDEXER_TRANSPORT as TransportType) || TransportType.Thread
+
+  const transportConfig: any =
+    tcpUrls || natsUrl ? { tcpUrls, natsUrl } : undefined
+
+  await SDK.init({
+    projectId,
+    transport,
+    transportConfig,
+    apiPort,
+    fetcher: {
+      instances: 1,
+    },
+    parser: {
+      instances: 1,
+    },
+    indexer: {
+      dataPath,
+      main: {
+        apiSchemaPath,
+        domainPath: mainDomainPath,
+      },
+      worker: {
+        instances,
+        domainPath: workerDomainPath,
+      },
+    },
+  })
+}
+
+main()

packages/token-snapshot/src/api/index.ts

@@ -0,0 +1 @@
+export { default } from './schema.js'

packages/token-snapshot/src/api/resolvers.ts

@@ -0,0 +1,56 @@
+import { SPLTokenHolding, SPLTokenInfo } from '../types.js'
+import MainDomain from '../domain/main.js'
+
+export type TokenFilters = {
+  mint?: string
+}
+
+export type TokenEventsFilters = {
+  mint: string
+  account?: string
+  types?: string[]
+  startDate?: number
+  endDate?: number
+  limit?: number
+  skip?: number
+  reverse?: boolean
+}
+
+export type TokenHoldersFilters = {
+  mint: string
+  timestamp?: number
+  limit?: number
+  skip?: number
+  reverse?: boolean
+  gte?: string
+  lte?: string
+}
+
+export class APIResolver {
+  constructor(protected domain: MainDomain) {}
+
+  async getTokens(mint?: string): Promise<SPLTokenInfo[]> {
+    return await this.filterTokens({ mint })
+  }
+
+  async getTokenHoldings(
+    filters: TokenHoldersFilters,
+  ): Promise<SPLTokenHolding[]> {
+    return await this.domain.getTokenHoldings(filters.mint, filters)
+  }
+
+  protected async filterTokens({
+    mint,
+  }: TokenFilters): Promise<SPLTokenInfo[]> {
+    const domTokens = await this.domain.getTokens()
+
+    const tokens =
+      [mint] || Object.values(domTokens).map((token: any) => token.address)
+
+    const result = (tokens as string[])
+      .map((address) => domTokens[address])
+      .filter((token) => !!token)
+
+    return result
+  }
+}

packages/token-snapshot/src/api/schema.ts

@@ -0,0 +1,49 @@
+import {
+  GraphQLObjectType,
+  GraphQLNonNull,
+  GraphQLString,
+  GraphQLList,
+  GraphQLInt,
+  GraphQLBoolean,
+} from 'graphql'
+import { GraphQLLong } from '@aleph-indexer/core'
+import { IndexerAPISchema } from '@aleph-indexer/framework'
+import * as Types from './types.js'
+import { APIResolver, TokenEventsFilters } from './resolvers.js'
+import MainDomain from '../domain/main.js'
+
+export default class APISchema extends IndexerAPISchema {
+  constructor(
+    protected domain: MainDomain,
+    protected resolver: APIResolver = new APIResolver(domain),
+  ) {
+    super(domain, {
+      query: new GraphQLObjectType({
+        name: 'Query',
+        fields: {
+          tokenMints: {
+            type: Types.TokenMints,
+            args: {
+              mint: { type: new GraphQLList(GraphQLString) },
+            },
+            resolve: (_, ctx) => this.resolver.getTokens(ctx.mint),
+          },
+          tokenHolders: {
+            type: Types.TokenHolders,
+            args: {
+              mint: { type: new GraphQLNonNull(GraphQLString) },
+              timestamp: { type: GraphQLLong },
+              limit: { type: GraphQLInt },
+              skip: { type: GraphQLInt },
+              reverse: { type: GraphQLBoolean },
+              gte: { type: GraphQLString },
+              lte: { type: GraphQLString },
+            },
+            resolve: (_, ctx) =>
+              this.resolver.getTokenHoldings(ctx as TokenEventsFilters),
+          },
+        },
+      }),
+    })
+  }
+}

packages/token-snapshot/src/api/types.ts

@@ -0,0 +1,56 @@
+import {
+  GraphQLObjectType,
+  GraphQLString,
+  GraphQLNonNull,
+  GraphQLList,
+} from 'graphql'
+import { TokenType, GraphQLBigNumber } from '@aleph-indexer/core'
+
+// ------------------- TOKENS --------------------------
+
+export const TokenMint = new GraphQLObjectType({
+  name: 'TokenMint',
+  fields: {
+    name: { type: new GraphQLNonNull(GraphQLString) },
+    address: { type: new GraphQLNonNull(GraphQLString) },
+    programId: { type: new GraphQLNonNull(GraphQLString) },
+    tokenInfo: { type: TokenType },
+  },
+})
+
+export const TokenMints = new GraphQLList(TokenMint)
+
+// ------------------- HOLDERS --------------------------
+export const LendingBalance = new GraphQLObjectType({
+  name: 'LendingBalance',
+  fields: {
+    deposited: { type: new GraphQLNonNull(GraphQLBigNumber) },
+    borrowed: { type: new GraphQLNonNull(GraphQLBigNumber) },
+  },
+})
+
+export const TokenBalances = new GraphQLObjectType({
+  name: 'TokenBalances',
+  fields: {
+    wallet: { type: new GraphQLNonNull(GraphQLBigNumber) },
+    solend: { type: new GraphQLNonNull(LendingBalance) },
+    port: { type: new GraphQLNonNull(LendingBalance) },
+    larix: { type: new GraphQLNonNull(LendingBalance) },
+    total: { type: new GraphQLNonNull(GraphQLBigNumber) },
+  },
+})
+
+// ------------------- HOLDERS --------------------------
+
+export const TokenHolder = new GraphQLObjectType({
+  name: 'TokenHolder',
+  fields: {
+    account: { type: new GraphQLNonNull(GraphQLString) },
+    owner: { type: new GraphQLNonNull(GraphQLString) },
+    tokenMint: { type: new GraphQLNonNull(GraphQLString) },
+    balances: { type: new GraphQLNonNull(TokenBalances) },
+    timestamp: { type: new GraphQLNonNull(GraphQLString) },
+  },
+})
+
+export const TokenHolders = new GraphQLList(TokenHolder)

packages/token-snapshot/src/constants.ts

@@ -0,0 +1,8 @@
+import { constants } from '@aleph-indexer/core'
+
+const { TOKEN_PROGRAM_ID, TOKEN_PROGRAM_ID_PK } = constants
+export { TOKEN_PROGRAM_ID, TOKEN_PROGRAM_ID_PK }
+
+export enum ProgramName {
+  TokenSnapshot = 'token-snapshot',
+}

packages/token-snapshot/src/dal/accountMints.ts

@@ -0,0 +1,22 @@
+import { EntityStorage } from '@aleph-indexer/core'
+import { AccountMint } from '../domain/types.js'
+
+export type AccountMintStorage = EntityStorage<AccountMint>
+
+const mintKey = {
+  get: (e: AccountMint) => e.mint,
+  length: EntityStorage.AddressLength,
+}
+
+const accountKey = {
+  get: (e: AccountMint) => e.account,
+  length: EntityStorage.AddressLength,
+}
+
+export function createAccountMintDAL(path: string): AccountMintStorage {
+  return new EntityStorage<AccountMint>({
+    name: 'account_mint',
+    path,
+    key: [mintKey, accountKey],
+  })
+}

packages/token-snapshot/src/dal/balanceHistory.ts

@@ -0,0 +1,43 @@
+import { EntityStorage } from '@aleph-indexer/core'
+import { SPLTokenHolding } from '../types.js'
+import { getBigNumberMapFn } from './common.js'
+
+const mappedProps = ['deposited', 'borrowed', 'wallet', 'total']
+
+export enum BalanceHistoryDALIndex {
+  MintAccount = 'mint_account',
+}
+
+export type AccountBalanceHistoryStorage = EntityStorage<SPLTokenHolding>
+
+const mintKey = {
+  get: (e: SPLTokenHolding) => e.tokenMint,
+  length: EntityStorage.AddressLength,
+}
+
+const accountKey = {
+  get: (e: SPLTokenHolding) => e.account,
+  length: EntityStorage.AddressLength,
+}
+
+const timestampKey = {
+  get: (e: SPLTokenHolding) => e.timestamp,
+  length: EntityStorage.TimestampLength,
+}
+
+export function createBalanceHistoryDAL(
+  path: string,
+): AccountBalanceHistoryStorage {
+  return new EntityStorage<SPLTokenHolding>({
+    name: 'account_balance_history',
+    path,
+    key: [mintKey, accountKey, timestampKey],
+    indexes: [
+      {
+        name: BalanceHistoryDALIndex.MintAccount,
+        key: [mintKey, accountKey],
+      },
+    ],
+    mapFn: getBigNumberMapFn(mappedProps),
+  })
+}

packages/token-snapshot/src/dal/balanceState.ts

@@ -0,0 +1,47 @@
+import { EntityStorage, EntityUpdateOp } from '@aleph-indexer/core'
+import { SPLTokenHolding } from '../types.js'
+import { getBigNumberMapFn } from './common.js'
+
+const mappedProps = ['deposited', 'borrowed', 'wallet', 'total']
+
+export enum BalanceStateDALIndex {
+  Mint = 'mint',
+}
+
+export type AccountBalanceStateStorage = EntityStorage<SPLTokenHolding>
+
+const accountKey = {
+  get: (e: SPLTokenHolding) => e.account,
+  length: EntityStorage.AddressLength,
+}
+
+const mintKey = {
+  get: (e: SPLTokenHolding) => e.tokenMint,
+  length: EntityStorage.AddressLength,
+}
+
+export function createBalanceStateDAL(
+  path: string,
+): AccountBalanceStateStorage {
+  return new EntityStorage<SPLTokenHolding>({
+    name: 'account_balance_state',
+    path,
+    key: [mintKey, accountKey],
+    indexes: [
+      {
+        name: BalanceStateDALIndex.Mint,
+        key: [mintKey],
+      },
+    ],
+    async updateCheckFn(
+      oldEntity: SPLTokenHolding | undefined,
+      newEntity: SPLTokenHolding,
+    ): Promise<EntityUpdateOp> {
+      if (oldEntity && oldEntity.timestamp > newEntity.timestamp) {
+        return EntityUpdateOp.Keep
+      }
+      return EntityUpdateOp.Update
+    },
+    mapFn: getBigNumberMapFn(mappedProps),
+  })
+}

packages/token-snapshot/src/dal/common.ts

@@ -0,0 +1,18 @@
+import BN from 'bn.js'
+
+export function getBigNumberMapFn(
+  mappedProps: string[],
+): (arg0: any) => Promise<any> {
+  return async function (entry: { key: any; value: any }) {
+    const { key, value } = entry
+
+    // @note: Stored as hex strings (bn.js "toJSON" method), so we need to cast them to BN always
+    for (const prop of mappedProps) {
+      if (!(prop in value)) continue
+      if ((value as any)[prop] instanceof BN) continue
+      ;(value as any)[prop] = new BN((value as any)[prop], 'hex')
+    }
+
+    return { key, value }
+  }
+}

packages/token-snapshot/src/dal/event.ts

@@ -0,0 +1,51 @@
+import { EntityStorage } from '@aleph-indexer/core'
+import { SPLTokenEvent } from '../types.js'
+import { getBigNumberMapFn } from './common.js'
+
+const mappedProps = ['balance']
+
+export type EventStorage = EntityStorage<SPLTokenEvent>
+
+export enum EventDALIndex {
+  AccountTimestamp = 'account_timestamp',
+  AccountTypeTimestamp = 'account_type_timestamp',
+}
+
+const idKey = {
+  get: (e: SPLTokenEvent) => e.id,
+  length: EntityStorage.VariableLength,
+}
+
+const accountKey = {
+  get: (e: SPLTokenEvent) => e.account,
+  length: EntityStorage.AddressLength,
+}
+
+const typeKey = {
+  get: (e: SPLTokenEvent) => e.type,
+  length: EntityStorage.VariableLength,
+}
+
+const timestampKey = {
+  get: (e: SPLTokenEvent) => e.timestamp,
+  length: EntityStorage.TimestampLength,
+}
+
+export function createEventDAL(path: string): EventStorage {
+  return new EntityStorage<SPLTokenEvent>({
+    name: 'token_event',
+    path,
+    key: [idKey],
+    indexes: [
+      {
+        name: EventDALIndex.AccountTimestamp,
+        key: [accountKey, timestampKey],
+      },
+      {
+        name: EventDALIndex.AccountTypeTimestamp,
+        key: [accountKey, typeKey, timestampKey],
+      },
+    ],
+    mapFn: getBigNumberMapFn(mappedProps),
+  })
+}

packages/token-snapshot/src/dal/fetchMint.ts

@@ -0,0 +1,36 @@
+import {
+  EntityUpdateOp,
+  PendingWork,
+  PendingWorkStorage,
+} from '@aleph-indexer/core'
+import { MintAccount } from '../domain/types.js'
+
+export type FetchMintStorage = PendingWorkStorage<MintAccount>
+
+/**
+ * Creates a new pending transaction storage for the fetcher.
+ * @param path Path to the database.
+ * @param name Name of the storage file
+ */
+export function createFetchMintDAL(
+  path: string,
+  name = 'fetcher_mint_accounts',
+): FetchMintStorage {
+  return new PendingWorkStorage<MintAccount>({
+    name,
+    path,
+    count: true,
+    async updateCheckFn(
+      oldEntity: PendingWork<MintAccount> | undefined,
+      newEntity: PendingWork<MintAccount>,
+    ): Promise<EntityUpdateOp> {
+      if (oldEntity) {
+        if (oldEntity.payload.timestamp > newEntity.payload.timestamp) {
+          newEntity.payload = oldEntity.payload
+        }
+      }
+
+      return EntityUpdateOp.Update
+    },
+  })
+}

packages/token-snapshot/src/domain/account.ts

@@ -0,0 +1,41 @@
+import { EventDALIndex, EventStorage } from '../dal/event.js'
+import { SPLTokenEvent } from '../types.js'
+import { AccountEventsFilters } from './types.js'
+
+export class Account {
+  constructor(public address: string, protected eventDAL: EventStorage) {}
+
+  async getEvents(filters: AccountEventsFilters): Promise<SPLTokenEvent[]> {
+    const { startDate, endDate, types, skip: sk, ...opts } = filters
+
+    const typesMap = types ? new Set(types) : undefined
+
+    let skip = sk || 0
+    const limit = opts.limit || 1000
+    opts.limit = !typesMap ? limit + skip : undefined
+
+    const result: SPLTokenEvent[] = []
+
+    const from = startDate ? [this.address, startDate] : [this.address]
+    const to = endDate ? [this.address, endDate] : [this.address]
+
+    const events = await this.eventDAL
+      .useIndex(EventDALIndex.AccountTimestamp)
+      .getAllFromTo(from, to, opts)
+
+    for await (const { value } of events) {
+      // @note: Filter by type
+      if (typesMap && !typesMap.has(value.type)) continue
+
+      // @note: Skip first N events
+      if (--skip >= 0) continue
+
+      result.push(value)
+
+      // @note: Stop when after reaching the limit
+      if (limit > 0 && result.length >= limit) return result
+    }
+
+    return result
+  }
+}

packages/token-snapshot/src/domain/main.ts

@@ -0,0 +1,118 @@
+import {
+  AccountIndexerRequestArgs,
+  IndexerMainDomain,
+  IndexerMainDomainContext,
+  IndexerMainDomainWithDiscovery,
+} from '@aleph-indexer/framework'
+import { Token, solanaPrivateRPCRoundRobin } from '@aleph-indexer/core'
+import { SPLTokenHolding, SPLTokenInfo, SPLTokenType } from '../types.js'
+import { discoveryFn } from '../utils/discovery.js'
+import { TokenHoldersFilters } from './types.js'
+import { TOKEN_PROGRAM_ID } from '../constants.js'
+
+export default class MainDomain
+  extends IndexerMainDomain
+  implements IndexerMainDomainWithDiscovery
+{
+  protected tokens: Record<string, SPLTokenInfo> = {}
+
+  constructor(protected context: IndexerMainDomainContext) {
+    super(context, {})
+  }
+
+  async discoverAccounts(): Promise<AccountIndexerRequestArgs[]> {
+    const init = {
+      account: '',
+      index: {
+        transactions: {
+          chunkDelay: 0,
+          chunkTimeframe: 1000 * 60 * 60 * 24,
+        },
+        content: false,
+      },
+    }
+    return [init]
+  }
+
+  async init(...args: unknown[]): Promise<void> {
+    await super.init(...args)
+    const { accounts, mints } = await discoveryFn()
+
+    await Promise.all(
+      accounts.map(async (account: string) => {
+        const connection = solanaPrivateRPCRoundRobin.getClient()
+        const mint = await Token.getTokenMintByAccount(
+          account,
+          connection.getConnection(),
+        )
+        await this.addToken(mint)
+
+        const options = {
+          account,
+          meta: { type: SPLTokenType.Account, mint: mint },
+          index: {
+            transactions: {
+              chunkDelay: 0,
+              chunkTimeframe: 1000 * 60 * 60 * 24,
+            },
+            content: false,
+          },
+        }
+        await this.context.apiClient.indexAccount(options)
+        this.accounts.add(account)
+      }),
+    )
+    await Promise.all(
+      mints.map(async (mint: string) => {
+        await this.addToken(mint)
+        const options = {
+          account: mint,
+          meta: { type: SPLTokenType.Mint, mint },
+          index: {
+            transactions: {
+              chunkDelay: 0,
+              chunkTimeframe: 1000 * 60 * 60 * 24,
+            },
+            content: false,
+          },
+        }
+        await this.context.apiClient.indexAccount(options)
+        this.accounts.add(mint)
+      }),
+    )
+  }
+
+  async getTokens(): Promise<Record<string, SPLTokenInfo>> {
+    return this.tokens
+  }
+
+  async getTokenHoldings(
+    account: string,
+    filters: TokenHoldersFilters,
+  ): Promise<SPLTokenHolding[]> {
+    return (await this.context.apiClient.invokeDomainMethod({
+      account,
+      args: [filters],
+      method: 'getTokenHoldings',
+    })) as SPLTokenHolding[]
+  }
+
+  protected async addToken(mint: string): Promise<void> {
+    const connection = solanaPrivateRPCRoundRobin.getClient()
+    const tokenInfo = await Token.getTokenByAddress(
+      mint,
+      connection.getConnection(),
+    )
+
+    if (!tokenInfo) return
+
+    const entity: SPLTokenInfo = {
+      name: tokenInfo.symbol,
+      address: mint,
+      programId: TOKEN_PROGRAM_ID,
+      tokenInfo,
+    }
+
+    this.tokens[mint] = entity
+  }
+}

packages/token-snapshot/src/domain/mint.ts

@@ -0,0 +1,118 @@
+import { StorageValueStream } from '@aleph-indexer/core'
+import { EventStorage } from '../dal/event.js'
+import { AccountMint, TokenHoldersFilters } from './types.js'
+import { SPLTokenHolding } from '../types.js'
+import BN from 'bn.js'
+import {
+  BalanceStateDALIndex,
+  AccountBalanceStateStorage,
+} from '../dal/balanceState.js'
+import { AccountMintStorage } from '../dal/accountMints.js'
+import {
+  AccountBalanceHistoryStorage,
+  BalanceHistoryDALIndex,
+} from '../dal/balanceHistory.js'
+
+export class Mint {
+  constructor(
+    protected address: string,
+    protected eventDAL: EventStorage,
+    protected balanceStateDAL: AccountBalanceStateStorage,
+    protected balanceHistoryDAL: AccountBalanceHistoryStorage,
+    protected accountMintDAL: AccountMintStorage,
+  ) {}
+
+  async getMintAccounts(
+    account?: string,
+  ): Promise<StorageValueStream<AccountMint>> {
+    const range = account ? [this.address, account] : [this.address]
+    return await this.accountMintDAL.getAllValuesFromTo(range, range)
+  }
+
+  async addAccount(account: string): Promise<void> {
+    const accountMint: AccountMint = {
+      mint: this.address,
+      account,
+    }
+    await this.accountMintDAL.save(accountMint)
+  }
+
+  async getTokenHoldings({
+    timestamp,
+    limit,
+    skip = 0,
+    reverse = true,
+    gte,
+    lte,
+  }: TokenHoldersFilters): Promise<SPLTokenHolding[]> {
+    // @note: Default limit and gte
+    limit = limit || 1000
+
+    // @note: Do not add constraints to limit arg when it is ALEPH token
+    if (limit < 1 || limit > 1000)
+      throw new Error('400 Bad Request: 1 <= limit <= 1000')
+
+    const gteBn = gte ? new BN(gte) : undefined
+    const lteBn = lte ? new BN(lte) : undefined
+
+    const result: SPLTokenHolding[] = []
+    const currentBalances = await this.balanceStateDAL
+      .useIndex(BalanceStateDALIndex.Mint)
+      .getAllValuesFromTo([this.address], [this.address], {
+        reverse,
+        limit,
+      })
+
+    if (timestamp) {
+      for await (const value of currentBalances) {
+        let snapshotBalance = await this.balanceHistoryDAL.getLastValueFromTo(
+          [this.address, value.account, 0],
+          [this.address, value.account, timestamp],
+        )
+        snapshotBalance = this.filterBalance(snapshotBalance, gteBn, lteBn)
+        if (!snapshotBalance) continue
+
+        // @note: Skip first N events
+        if (--skip >= 0) continue
+
+        result.push(snapshotBalance)
+
+        // @note: Stop when after reaching the limit
+        if (limit > 0 && result.length >= limit) return result
+      }
+    } else {
+      for await (const value of currentBalances) {
+        const balance = this.filterBalance(value, gteBn, lteBn)
+        if (!balance) continue
+
+        // @note: Skip first N events
+        if (--skip >= 0) continue
+
+        result.push(value)
+
+        // @note: Stop when after reaching the limit
+        if (limit > 0 && result.length >= limit) return result
+      }
+    }
+
+    return result
+  }
+
+  private filterBalance(
+    balance: SPLTokenHolding | undefined,
+    gteBn: BN | undefined,
+    lteBn: BN | undefined,
+  ): SPLTokenHolding | undefined {
+    if (!balance) return undefined
+
+    // @note: Filter by gte || lte
+    if (gteBn || lteBn) {
+      const balanceBN = new BN(balance.balances.total)
+
+      if (gteBn && balanceBN.lte(gteBn)) return undefined
+      if (lteBn && balanceBN.gte(lteBn)) return undefined
+    }
+
+    return balance
+  }
+}

packages/token-snapshot/src/domain/types.ts

@@ -0,0 +1,56 @@
+import { SPLTokenEvent } from '../types.js'
+
+export type MintEventsFilters = {
+  account?: string
+  startDate?: number
+  endDate?: number
+  types?: string[]
+  limit?: number
+  reverse?: boolean
+  skip?: number
+}
+
+export type AccountEventsFilters = {
+  startDate?: number
+  endDate?: number
+  types?: string[]
+  limit?: number
+  reverse?: boolean
+  skip?: number
+}
+
+export type TokenHoldersFilters = {
+  timestamp?: number
+  limit?: number
+  skip?: number
+  reverse?: boolean
+  gte?: string
+  lte?: string
+}
+
+export type AccountHoldingsFilters = {
+  account?: string
+  startDate?: number
+  endDate?: number
+  gte?: string
+  lte?: string
+}
+
+export type AccountHoldingsOptions = {
+  startDate?: number
+  endDate?: number
+  reverse?: boolean
+  limit?: number
+  skip?: number
+}
+
+export type MintAccount = {
+  mint: string
+  timestamp: number
+  event: SPLTokenEvent
+}
+
+export type AccountMint = {
+  account: string
+  mint: string
+}

packages/token-snapshot/src/domain/worker.ts

@@ -0,0 +1,302 @@
+import BN from 'bn.js'
+import {
+  AccountIndexerConfigWithMeta,
+  AccountStatsFilters,
+  createStatsStateDAL,
+  createStatsTimeSeriesDAL,
+  IndexerDomainContext,
+  IndexerWorkerDomain,
+  IndexerWorkerDomainWithStats,
+  InstructionContextV1,
+} from '@aleph-indexer/framework'
+import { PendingWork, PendingWorkPool } from '@aleph-indexer/core'
+import {
+  createEventParser,
+  TokenEventParser as eventParser,
+} from '../parsers/tokenEvent.js'
+import { mintParser as mParser } from '../parsers/mint.js'
+import { createEventDAL } from '../dal/event.js'
+import {
+  SPLTokenHolding,
+  SPLTokenAccount,
+  SPLTokenEvent,
+  SPLTokenEventType,
+  SPLTokenType,
+} from '../types.js'
+import { Mint } from './mint.js'
+import { TOKEN_PROGRAM_ID } from '../constants.js'
+import {
+  getBalanceFromEvent,
+  getEventAccounts,
+  isSPLTokenInstruction,
+} from '../utils/utils.js'
+import { createFetchMintDAL } from '../dal/fetchMint.js'
+import { MintAccount, TokenHoldersFilters } from './types.js'
+import { createBalanceHistoryDAL } from '../dal/balanceHistory.js'
+import { createBalanceStateDAL } from '../dal/balanceState.js'
+import { createAccountMintDAL } from '../dal/accountMints.js'
+import assert from 'assert'
+
+export default class WorkerDomain
+  extends IndexerWorkerDomain
+  implements IndexerWorkerDomainWithStats
+{
+  public mints: Record<string, Mint> = {}
+  public accountMints: PendingWorkPool<MintAccount>
+
+  constructor(
+    protected context: IndexerDomainContext,
+    protected eventParser: eventParser,
+    protected mintParser = mParser,
+    protected eventDAL = createEventDAL(context.dataPath),
+    protected statsStateDAL = createStatsStateDAL(context.dataPath),
+    protected statsTimeSeriesDAL = createStatsTimeSeriesDAL(context.dataPath),
+    protected fetchMintDAL = createFetchMintDAL(context.dataPath),
+    protected balanceHistoryDAL = createBalanceHistoryDAL(context.dataPath),
+    protected balanceStateDAL = createBalanceStateDAL(context.dataPath),
+    protected accountMintDAL = createAccountMintDAL(context.dataPath),
+    protected programId = TOKEN_PROGRAM_ID,
+  ) {
+    super(context)
+    this.eventParser = createEventParser(this.fetchMintDAL, this.eventDAL)
+    this.accountMints = new PendingWorkPool<MintAccount>({
+      id: 'mintAccounts',
+      interval: 0,
+      chunkSize: 100,
+      concurrency: 1,
+      dal: this.fetchMintDAL,
+      handleWork: this._handleMintAccounts.bind(this),
+      checkComplete: () => false,
+    })
+  }
+
+  async init(): Promise<void> {
+    return
+  }
+
+  async onNewAccount(
+    config: AccountIndexerConfigWithMeta<SPLTokenAccount>,
+  ): Promise<void> {
+    const { account, meta } = config
+
+    if (
+      meta.type === SPLTokenType.Mint ||
+      meta.type === SPLTokenType.Account ||
+      meta.type === SPLTokenType.AccountMint
+    ) {
+      const mint = meta.mint
+      if (!this.mints[mint]) {
+        this.mints[mint] = new Mint(
+          mint,
+          this.eventDAL,
+          this.balanceStateDAL,
+          this.balanceHistoryDAL,
+          this.accountMintDAL,
+        )
+      }
+      await this.mints[mint].addAccount(account)
+    }
+    console.log('Account indexing', this.context.instanceName, account)
+  }
+
+  async getTimeSeriesStats(
+    account: string,
+    type: string,
+    filters: AccountStatsFilters,
+  ): Promise<any> {
+    return {}
+  }
+
+  async getStats(account: string): Promise<any> {
+    return {}
+  }
+
+  async updateStats(account: string, now: number): Promise<void> {
+    console.log('', account)
+  }
+
+  async getTokenHoldings(
+    account: string,
+    filters: TokenHoldersFilters,
+  ): Promise<SPLTokenHolding[]> {
+    const mint = this.mints[account]
+    if (!mint) return []
+    return await mint.getTokenHoldings(filters)
+  }
+
+  protected async filterInstructions(
+    ixsContext: InstructionContextV1[],
+  ): Promise<InstructionContextV1[]> {
+    return ixsContext.filter(({ ix }) => isSPLTokenInstruction(ix))
+  }
+
+  protected async indexInstructions(
+    ixsContext: InstructionContextV1[],
+  ): Promise<void> {
+    const parsedEvents: SPLTokenEvent[] = []
+    const works: PendingWork<MintAccount>[] = []
+    const promises = ixsContext.map(async (ix) => {
+      const account = ix.txContext.parserContext.account
+      if (this.mints[account]) {
+        const parsedIx = this.mintParser.parse(ix, account)
+        if (parsedIx) {
+          if (parsedIx.type === SPLTokenEventType.InitializeAccount) {
+            const work = {
+              id: parsedIx.account,
+              time: Date.now(),
+              payload: {
+                mint: account,
+                timestamp: parsedIx.timestamp,
+                event: parsedIx,
+              },
+            }
+            works.push(work)
+          }
+        }
+      } else {
+        const parsedIx = await this.eventParser.parse(ix)
+        if (parsedIx) {
+          if (parsedIx.type === SPLTokenEventType.CloseAccount) {
+            const work = await this.fetchMintDAL.getFirstValueFromTo(
+              [parsedIx.account],
+              [parsedIx.account],
+              { atomic: true },
+            )
+            if (work && parsedIx.timestamp >= work.payload.timestamp) {
+              await this.accountMints.removeWork(work)
+              const options = {
+                account: parsedIx.account,
+                index: {
+                  transactions: true,
+                  content: true,
+                },
+              }
+              await this.context.apiClient.deleteAccount(options)
+            }
+          }
+          parsedEvents.push(parsedIx)
+        }
+      }
+    })
+
+    await Promise.all(promises)
+
+    console.log(`indexing ${ixsContext.length} parsed ixs`)
+
+    if (parsedEvents.length > 0) {
+      await this.eventDAL.save(parsedEvents)
+
+      for (const parsedEvent of parsedEvents) {
+        await this.dealWalletBalances(parsedEvent)
+      }
+    }
+    if (works.length > 0) {
+      await this.accountMints.addWork(works)
+    }
+  }
+
+  /**
+   * Fetch signatures from accounts.
+   * @param works Txn signatures with extra properties as time and payload.
+   */
+  protected async _handleMintAccounts(
+    works: PendingWork<MintAccount>[],
+  ): Promise<void> {
+    console.log(
+      `Mint accounts | Start handling ${works.length} minted accounts`,
+    )
+
+    for (const work of works) {
+      if (!work) continue
+
+      const account = work.id
+      const options = {
+        account,
+        meta: {
+          address: account,
+          type: SPLTokenType.AccountMint,
+          mint: work.payload.mint,
+        },
+        index: {
+          transactions: {
+            chunkDelay: 0,
+            chunkTimeframe: 1000 * 60 * 60 * 24,
+          },
+          content: false,
+        },
+      }
+      await this.context.apiClient.indexAccount(options)
+    }
+  }
+
+  protected async dealLendingBalances(entity: SPLTokenEvent): Promise<void> {
+    const accounts = getEventAccounts(entity)
+    const entities = accounts.map((account) => {
+      const balance = getBalanceFromEvent(entity, account)
+      return {
+        account,
+        tokenMint: entity.mint,
+        owner: entity.owner,
+        balances: {
+          wallet: balance,
+          total: balance,
+        },
+        timestamp: entity.timestamp,
+      } as SPLTokenHolding
+    })
+    await this.balanceHistoryDAL.save(entities)
+    await this.balanceStateDAL.save(entities)
+  }
+
+  protected async dealWalletBalances(
+    entity: SPLTokenEvent,
+  ): Promise<SPLTokenHolding[]> {
+    const accounts = getEventAccounts(entity)
+    const entities = await Promise.all(
+      accounts.map(async (account) => {
+        const balance = getBalanceFromEvent(entity, account)
+        const previousBalance = await this.balanceHistoryDAL.getLastValueFromTo(
+          [entity.mint, account, 0],
+          [entity.mint, account, entity.timestamp],
+        )
+        const solend = {
+          deposited: previousBalance?.balances.solend.deposited || '0',
+          borrowed: previousBalance?.balances.solend.borrowed || '0',
+        }
+        const port = {
+          deposited: previousBalance?.balances.port.deposited || '0',
+          borrowed: previousBalance?.balances.port.borrowed || '0',
+        }
+        const larix = {
+          deposited: previousBalance?.balances.larix.deposited || '0',
+          borrowed: previousBalance?.balances.larix.borrowed || '0',
+        }
+        // @note: The total is the sum of the wallet balance and the deposited amounts. The borrowed amounts are subtracted.
+        const total = new BN(balance)
+          .add(new BN(solend.deposited))
+          .add(new BN(port.deposited))
+          .add(new BN(larix.deposited))
+          .sub(new BN(solend.borrowed))
+          .sub(new BN(port.borrowed))
+          .sub(new BN(larix.borrowed))
+          .toString()
+        return {
+          account,
+          tokenMint: entity.mint,
+          owner: entity.owner,
+          balances: {
+            wallet: balance,
+            solend,
+            port,
+            larix,
+            total: total,
+          },
+          timestamp: entity.timestamp,
+        } as SPLTokenHolding
+      }),
+    )
+    await this.balanceHistoryDAL.save(entities)
+    await this.balanceStateDAL.save(entities)
+    return entities
+  }
+}

packages/token-snapshot/src/parsers/lendingEvent.ts

@@ -0,0 +1,286 @@
+import BN from 'bn.js'
+import {
+  InstructionContextV1,
+  Utils,
+  AlephParsedEvent,
+} from '@aleph-indexer/core'
+import { isMaxU64 } from '@aleph-indexer/layout'
+import {
+  LendingEvent,
+  BorrowObligationLiquidityEvent,
+  DepositObligationCollateralEvent,
+  DepositReserveLiquidityEvent,
+  LiquidateObligationEvent,
+  RedeemReserveCollateralEvent,
+  RepayObligationLiquidityEvent,
+  LendingEventType,
+  WithdrawObligationCollateralEvent,
+  LendingEventInfo,
+  DepositReserveLiquidityAndObligationCollateralEventInfo,
+  RedeemReserveCollateralEventInfo,
+  DepositObligationCollateralEventInfo,
+  WithdrawObligationCollateralEventInfo,
+  BorrowObligationLiquidityEventInfo,
+  RepayObligationLiquidityEventInfo,
+  LiquidateObligationEventInfo,
+  WithdrawObligationCollateralAndRedeemReserveCollateralEventInfo,
+  WithdrawObligationCollateralAndRedeemReserveCollateralEvent,
+  FlashLoanEventInfo,
+  FlashLoanEvent,
+} from '../types.js'
+
+const {
+  getTokenBalance,
+  getBurnedCollateralAmount,
+  getMintedCollateralAmount,
+  getSubInstructions,
+  getTransferedAmount,
+} = Utils
+
+export class EventParser {
+  parse(ixCtx: InstructionContextV1): LendingEvent {
+    const { ix, parentIx, txContext } = ixCtx
+    const { tx: parentTx } = txContext
+
+    const parsed = (ix as AlephParsedEvent<LendingEventType, LendingEventInfo>)
+      .parsed
+
+    const id = `${parentTx.signature}${
+      parentIx ? `:${parentIx.index.toString().padStart(2, '0')}` : ''
+    }:${ix.index.toString().padStart(2, '0')}`
+
+    const timestamp = parentTx.blockTime
+      ? parentTx.blockTime * 1000
+      : parentTx.slot
+
+    const baseEvent = {
+      ...parsed.info,
+      id,
+      timestamp,
+      type: parsed.type,
+    }
+
+    try {
+      switch (parsed.type) {
+        case LendingEventType.DepositReserveLiquidity:
+        case LendingEventType.DepositReserveLiquidityAndObligationCollateral: {
+          const subIxs = getSubInstructions(ixCtx)
+          const info =
+            parsed.info as DepositReserveLiquidityAndObligationCollateralEventInfo
+          const collateralAmount = getMintedCollateralAmount(
+            info.userCollateral,
+            info.reserveCollateralMint,
+            subIxs,
+          )
+          let liquidityAmount = info.liquidityAmount
+          if (isMaxU64(liquidityAmount)) {
+            liquidityAmount = getTransferedAmount(
+              info.userLiquidity,
+              info.reserveLiquidityVault,
+              subIxs,
+            )
+          }
+          const reserveLiquidityAmount = new BN(
+            getTokenBalance(parentTx, info.reserveLiquidityVault, true) || 0,
+          )
+          return {
+            ...baseEvent,
+            liquidityAmount,
+            collateralAmount,
+            reserveLiquidityAmount,
+          } as DepositReserveLiquidityEvent
+        }
+
+        case LendingEventType.RedeemReserveCollateral: {
+          const subIxs = getSubInstructions(ixCtx)
+          const info = parsed.info as RedeemReserveCollateralEventInfo
+          const liquidityAmount = getTransferedAmount(
+            info.reserveLiquidityVault,
+            info.userLiquidity,
+            subIxs,
+          )
+          let collateralAmount = info.collateralAmount
+          if (isMaxU64(collateralAmount)) {
+            collateralAmount = getBurnedCollateralAmount(
+              info.userLiquidity,
+              info.reserveLiquidityVault,
+              subIxs,
+            )
+          }
+          const reserveLiquidityAmount = new BN(
+            getTokenBalance(parentTx, info.reserveLiquidityVault, true) || 0,
+          )
+          return {
+            ...baseEvent,
+            liquidityAmount,
+            collateralAmount,
+            reserveLiquidityAmount,
+          } as RedeemReserveCollateralEvent
+        }
+
+        case LendingEventType.DepositObligationCollateral: {
+          const info = parsed.info as DepositObligationCollateralEventInfo
+          let collateralAmount = info.collateralAmount
+          if (isMaxU64(collateralAmount)) {
+            const subIxs = getSubInstructions(ixCtx)
+            collateralAmount = getTransferedAmount(
+              info.userCollateral,
+              info.reserveCollateralVault,
+              subIxs,
+            )
+          }
+          return {
+            ...baseEvent,
+            collateralAmount,
+          } as DepositObligationCollateralEvent
+        }
+
+        case LendingEventType.WithdrawObligationCollateralAndRedeemReserveCollateral:
+        case LendingEventType.WithdrawObligationCollateral: {
+          const info = parsed.info as WithdrawObligationCollateralEventInfo
+          let collateralAmount = info.collateralAmount
+          if (isMaxU64(collateralAmount)) {
+            const subIxs = getSubInstructions(ixCtx)
+            collateralAmount = getTransferedAmount(
+              info.reserveCollateralVault,
+              info.userCollateral,
+              subIxs,
+            )
+          }
+          if (parsed.type === LendingEventType.WithdrawObligationCollateral) {
+            return {
+              ...baseEvent,
+              collateralAmount,
+            } as WithdrawObligationCollateralEvent
+          } else {
+            const subIxs = getSubInstructions(ixCtx)
+            const info =
+              parsed.info as WithdrawObligationCollateralAndRedeemReserveCollateralEventInfo
+            const liquidityAmount = getTransferedAmount(
+              info.reserveLiquidityVault,
+              info.userLiquidity,
+              subIxs,
+            )
+            const reserveLiquidityAmount = new BN(
+              getTokenBalance(parentTx, info.reserveLiquidityVault, true) || 0,
+            )
+            return {
+              ...baseEvent,
+              collateralAmount,
+              liquidityAmount,
+              reserveLiquidityAmount,
+            } as unknown as WithdrawObligationCollateralAndRedeemReserveCollateralEvent
+          }
+        }
+
+        case LendingEventType.BorrowObligationLiquidity: {
+          const subIxs = getSubInstructions(ixCtx)
+          const info = parsed.info as BorrowObligationLiquidityEventInfo
+          const liquidityFeeAmount = getTransferedAmount(
+            info.reserveLiquidityVault,
+            info.liquidityFeeReceiver,
+            subIxs,
+          )
+          let liquidityAmount = info.liquidityAmount
+          if (isMaxU64(liquidityAmount)) {
+            liquidityAmount = getTransferedAmount(
+              info.reserveLiquidityVault,
+              info.userLiquidity,
+              subIxs,
+            )
+          }
+          const reserveLiquidityAmount = new BN(
+            getTokenBalance(parentTx, info.reserveLiquidityVault, true) || 0,
+          )
+          return {
+            ...baseEvent,
+            liquidityFeeAmount,
+            liquidityAmount,
+            reserveLiquidityAmount,
+          } as BorrowObligationLiquidityEvent
+        }
+
+        case LendingEventType.RepayObligationLiquidity: {
+          const info = parsed.info as RepayObligationLiquidityEventInfo
+          let liquidityAmount = info.liquidityAmount
+          if (isMaxU64(liquidityAmount)) {
+            const subIxs = getSubInstructions(ixCtx)
+            liquidityAmount = getTransferedAmount(
+              info.userLiquidity,
+              info.reserveLiquidityVault,
+              subIxs,
+            )
+          }
+          const reserveLiquidityAmount = new BN(
+            getTokenBalance(parentTx, info.reserveLiquidityVault, true) || 0,
+          )
+          return {
+            ...baseEvent,
+            liquidityAmount,
+            reserveLiquidityAmount,
+          } as RepayObligationLiquidityEvent
+        }
+
+        case LendingEventType.LiquidateObligation:
+        case LendingEventType.LiquidateObligation2: {
+          const subIxs = getSubInstructions(ixCtx)
+          const info = parsed.info as LiquidateObligationEventInfo
+          const liquidityRepayAmount = getTransferedAmount(
+            info.userLiquidity,
+            info.repayReserveLiquidityVault,
+            subIxs,
+          )
+          const collateralWithdrawAmount = getTransferedAmount(
+            info.withdrawReserveCollateralVault,
+            info.userCollateral,
+            subIxs,
+          )
+          const repayReserveLiquidityAmount = new BN(
+            getTokenBalance(parentTx, info.repayReserveLiquidityVault, true) ||
+              0,
+          )
+          return {
+            ...baseEvent,
+            liquidityRepayAmount,
+            collateralWithdrawAmount,
+            repayReserveLiquidityAmount,
+          } as LiquidateObligationEvent
+        }
+
+        case LendingEventType.FlashLoan: {
+          const subIxs = getSubInstructions(ixCtx)
+          const info = parsed.info as FlashLoanEventInfo
+          const liquidityFeeAmount = getTransferedAmount(
+            info.reserveLiquidityVault,
+            info.liquidityFeeReceiver,
+            subIxs,
+          )
+          let liquidityAmount = info.liquidityAmount
+          if (isMaxU64(liquidityAmount)) {
+            liquidityAmount = getTransferedAmount(
+              info.reserveLiquidityVault,
+              info.userLiquidity,
+              subIxs,
+            )
+          }
+          return {
+            ...baseEvent,
+            liquidityFeeAmount,
+            liquidityAmount,
+          } as FlashLoanEvent
+        }
+        case LendingEventType.InitReserve:
+        default: {
+          console.log('default -> ', parsed.type, id)
+          return baseEvent as LendingEvent
+        }
+      }
+    } catch (e) {
+      console.log('error -> ', parsed.type, id, e)
+      throw e
+    }
+  }
+}
+
+export const eventParser = new EventParser()
+export default eventParser

packages/token-snapshot/src/parsers/mint.ts

@@ -0,0 +1,83 @@
+import { InstructionContextV1, Utils } from '@aleph-indexer/core'
+import {
+  SPLTokenRawEvent,
+  SPLTokenEvent,
+  SPLTokenEventInitializeAccount,
+  SPLTokenEventInitializeMint,
+  SPLTokenEventType,
+} from '../types.js'
+
+const { getTokenBalance } = Utils
+
+export class MintParser {
+  parse(
+    ixCtx: InstructionContextV1,
+    mintAddress: string,
+  ): SPLTokenEvent | undefined {
+    const { ix, parentIx, txContext } = ixCtx
+    const parentTx = txContext.tx
+    const parsed = (ix as SPLTokenRawEvent).parsed
+
+    // @note: Skip unrelated token ixs from being parsed
+    if (
+      parsed.info &&
+      'mint' in parsed.info &&
+      parsed.info.mint !== mintAddress
+    )
+      return
+
+    const id = `${parentTx.signature}${
+      parentIx ? `:${parentIx.index.toString().padStart(2, '0')}` : ''
+    }:${ix.index.toString().padStart(2, '0')}`
+
+    const timestamp = parentTx.blockTime
+      ? parentTx.blockTime * 1000
+      : parentTx.slot
+    const type = parsed.type
+
+    switch (type) {
+      case SPLTokenEventType.InitializeAccount:
+      case SPLTokenEventType.InitializeAccount2:
+      case SPLTokenEventType.InitializeAccount3: {
+        const { account, owner, mint } = parsed.info
+        const balance = getTokenBalance(parentTx, account) as string
+
+        const res: SPLTokenEventInitializeAccount = {
+          id,
+          timestamp,
+          type: SPLTokenEventType.InitializeAccount,
+          balance,
+          account,
+          owner,
+          mint,
+        }
+
+        console.log('---> init account => ', account, id)
+
+        return res
+      }
+
+      case SPLTokenEventType.InitializeMint:
+      case SPLTokenEventType.InitializeMint2: {
+        const { mint, mintAuthority } = parsed.info
+
+        const res: SPLTokenEventInitializeMint = {
+          id,
+          timestamp,
+          type: SPLTokenEventType.InitializeMint,
+          balance: '0',
+          account: mintAuthority,
+          owner: mintAuthority,
+          mint,
+        }
+
+        console.log('---> init MINT => ', id)
+
+        return res
+      }
+    }
+  }
+}
+
+export const mintParser = new MintParser()
+export default mintParser

packages/token-snapshot/src/parsers/tokenEvent.ts

@@ -0,0 +1,353 @@
+import {
+  InstructionContextV1,
+  Utils,
+  solanaPrivateRPCRoundRobin,
+} from '@aleph-indexer/core'
+import { ParsedAccountData, PublicKey } from '@solana/web3.js'
+import { SPLTokenRawEvent, SPLTokenEvent, SPLTokenEventType } from '../types.js'
+import { getMintAndOwnerFromEvent } from '../utils/utils.js'
+import { FetchMintStorage } from '../dal/fetchMint.js'
+import { EventDALIndex, EventStorage } from '../dal/event.js'
+
+const { getTokenBalance } = Utils
+
+export type MintOwner = {
+  mint: string
+  owner: string
+}
+
+export class TokenEventParser {
+  constructor(
+    protected fetchMintDAL: FetchMintStorage,
+    protected eventDAL: EventStorage,
+  ) {}
+
+  async parse(ixCtx: InstructionContextV1): Promise<SPLTokenEvent | undefined> {
+    const { ix, parentIx, txContext } = ixCtx
+    const { tx: parentTx } = txContext
+
+    const parsed = (ix as SPLTokenRawEvent).parsed
+
+    const id = `${parentTx.signature}${
+      parentIx ? `:${parentIx.index.toString().padStart(2, '0')}` : ''
+    }:${ix.index.toString().padStart(2, '0')}`
+
+    const timestamp = parentTx.blockTime
+      ? parentTx.blockTime * 1000
+      : parentTx.slot
+    const type = parsed.type
+
+    switch (type) {
+      case SPLTokenEventType.MintTo: {
+        const account = parsed.info.account
+        const balance = getTokenBalance(parentTx, account) as string
+        const { mint, owner } = await this.getMintAndOwner(account)
+
+        return {
+          id,
+          timestamp,
+          type,
+          amount: parsed.info.amount,
+          balance,
+          account,
+          mint,
+          owner,
+        }
+      }
+      case SPLTokenEventType.MintToChecked: {
+        const { account, tokenAmount } = parsed.info
+        const balance = getTokenBalance(parentTx, account) as string
+        const { mint, owner } = await this.getMintAndOwner(account)
+
+        return {
+          id,
+          timestamp,
+          type: SPLTokenEventType.MintTo,
+          amount: tokenAmount.amount,
+          balance,
+          account,
+          mint,
+          owner,
+        }
+      }
+      case SPLTokenEventType.Burn: {
+        const { account, amount } = parsed.info
+        const balance = getTokenBalance(parentTx, account) as string
+        const { mint, owner } = await this.getMintAndOwner(account)
+
+        return {
+          id,
+          timestamp,
+          type,
+          amount,
+          balance,
+          account,
+          mint,
+          owner,
+        }
+      }
+      case SPLTokenEventType.BurnChecked: {
+        const { account, tokenAmount } = parsed.info
+        const balance = getTokenBalance(parentTx, account) as string
+        const { mint, owner } = await this.getMintAndOwner(account)
+
+        return {
+          id,
+          timestamp,
+          type: SPLTokenEventType.Burn,
+          amount: tokenAmount.amount,
+          balance,
+          account,
+          mint,
+          owner,
+        }
+      }
+      case SPLTokenEventType.InitializeAccount:
+      case SPLTokenEventType.InitializeAccount2:
+      case SPLTokenEventType.InitializeAccount3: {
+        const { account, owner, mint } = parsed.info
+        const balance = getTokenBalance(parentTx, account) as string
+
+        return {
+          id,
+          timestamp,
+          type: SPLTokenEventType.InitializeAccount,
+          balance,
+          account,
+          mint,
+          owner,
+        }
+      }
+      case SPLTokenEventType.CloseAccount: {
+        const { account, destination } = parsed.info
+        const owner =
+          'owner' in parsed.info ? parsed.info.owner : parsed.info.multisigOwner
+
+        const balance = getTokenBalance(parentTx, account) as string
+        const { mint } = await this.getMintAndOwner(account)
+
+        return {
+          id,
+          timestamp,
+          type,
+          balance,
+          account,
+          mint,
+          owner,
+          toAccount: destination,
+        }
+      }
+      case SPLTokenEventType.Transfer: {
+        const account = parsed.info.source
+        const balance = getTokenBalance(parentTx, account) as string
+
+        const toAccount = parsed.info.destination
+        const toBalance = getTokenBalance(parentTx, toAccount) as string
+
+        const { mint } = await this.getMintAndOwner(account)
+        const { owner: toOwner } = await this.getMintAndOwner(toAccount)
+
+        const owner =
+          'authority' in parsed.info
+            ? parsed.info.authority
+            : parsed.info.multisigAuthority
+
+        return {
+          id,
+          timestamp,
+          type,
+          amount: parsed.info.amount,
+          balance,
+          account,
+          owner,
+          toBalance,
+          toAccount,
+          toOwner,
+          mint,
+        }
+      }
+      case SPLTokenEventType.TransferChecked: {
+        const account = parsed.info.source
+        const balance = getTokenBalance(parentTx, account) as string
+
+        const toAccount = parsed.info.destination
+        const toBalance = getTokenBalance(parentTx, toAccount) as string
+
+        const { mint } = await this.getMintAndOwner(account)
+        const { owner: toOwner } = await this.getMintAndOwner(toAccount)
+
+        return {
+          id,
+          timestamp,
+          type: SPLTokenEventType.Transfer,
+          amount: parsed.info.tokenAmount.amount,
+          balance,
+          account,
+          owner: parsed.info.authority,
+          toBalance,
+          toAccount,
+          toOwner,
+          mint,
+        }
+      }
+      case SPLTokenEventType.SetAuthority: {
+        const { account, authority, authorityType, newAuthority } = parsed.info
+        const balance = getTokenBalance(parentTx, account) as string
+        const { mint, owner } = await this.getMintAndOwner(account)
+
+        return {
+          id,
+          timestamp,
+          type,
+          balance,
+          account,
+          owner,
+          newOwner: newAuthority,
+          authorityType,
+          mint,
+        }
+      }
+      case SPLTokenEventType.Approve: {
+        const { source: account, owner, delegate, amount } = parsed.info
+        const balance = getTokenBalance(parentTx, account) as string
+        const { mint } = await this.getMintAndOwner(account)
+
+        return {
+          id,
+          timestamp,
+          type,
+          amount,
+          balance,
+          account,
+          owner,
+          delegate,
+          mint,
+        }
+      }
+      case SPLTokenEventType.ApproveChecked: {
+        const { source: account, owner, delegate, tokenAmount } = parsed.info
+        const balance = getTokenBalance(parentTx, account) as string
+        const { mint } = await this.getMintAndOwner(account)
+
+        return {
+          id,
+          timestamp,
+          type: SPLTokenEventType.Approve,
+          amount: tokenAmount.amount,
+          balance,
+          account,
+          owner,
+          delegate,
+          mint,
+        }
+      }
+      case SPLTokenEventType.Revoke: {
+        const { source: account, owner } = parsed.info
+        const balance = getTokenBalance(parentTx, account) as string
+        const { mint } = await this.getMintAndOwner(account)
+
+        return {
+          id,
+          timestamp,
+          type,
+          balance,
+          account,
+          owner,
+          mint,
+        }
+      }
+      case SPLTokenEventType.SyncNative: {
+        const { account } = parsed.info
+        const balance = getTokenBalance(parentTx, account) as string
+        const { mint, owner } = await this.getMintAndOwner(account)
+
+        return {
+          id,
+          timestamp,
+          type,
+          balance,
+          account,
+          owner,
+          mint,
+        }
+      }
+
+      default: {
+        console.log('NOT PARSED IX TYPE', (parsed as any).type)
+        console.log(id)
+        return
+      }
+    }
+  }
+
+  protected async getMintAndOwner(account: string): Promise<MintOwner> {
+    const dbEvent = await this.fetchMintDAL.getFirstValueFromTo(
+      [account],
+      [account],
+      { atomic: true },
+    )
+    if (!dbEvent) {
+      return await this.getMintAndOwnerFromEvents(account)
+    }
+
+    const event = dbEvent.payload.event
+    const data = getMintAndOwnerFromEvent(event, account)
+
+    return {
+      mint: data.mint,
+      owner: data.owner || '',
+    }
+  }
+
+  protected async getMintAndOwnerFromEvents(
+    account: string,
+  ): Promise<MintOwner> {
+    const dbEvent = await this.eventDAL
+      .useIndex(EventDALIndex.AccountTimestamp)
+      .getFirstValueFromTo([account], [account], { atomic: true })
+
+    if (!dbEvent) {
+      return await this.getMintAndOwnerFromAccount(account)
+    }
+
+    const data = getMintAndOwnerFromEvent(dbEvent, account)
+
+    return {
+      mint: data.mint,
+      owner: data.owner || '',
+    }
+  }
+
+  protected async getMintAndOwnerFromAccount(
+    account: string,
+  ): Promise<MintOwner> {
+    // TODO: Improve this way to get the mint and owner of an account
+    try {
+      const connection = solanaPrivateRPCRoundRobin.getClient()
+      const res = await connection
+        .getConnection()
+        .getParsedAccountInfo(new PublicKey(account))
+
+      const data = (res?.value?.data as ParsedAccountData)?.parsed?.info
+
+      return {
+        mint: data.mint,
+        owner: data.owner,
+      }
+    } catch (e) {
+      console.log('Error checking info for account ' + account)
+    }
+
+    return {
+      mint: '',
+      owner: '',
+    }
+  }
+}
+
+export function createEventParser(
+  fetchDAL: FetchMintStorage,
+  eventDAL: EventStorage,
+): TokenEventParser {
+  return new TokenEventParser(fetchDAL, eventDAL)
+}

packages/token-snapshot/src/utils/discovery.ts

@@ -0,0 +1,96 @@
+import fs, { Stats } from 'fs'
+import path from 'path'
+import { config } from '@aleph-indexer/core'
+import { DiscoveryFn, DiscoveryFnReturn } from '../types.js'
+
+export async function discoveryFn(): Promise<DiscoveryFnReturn> {
+  // @note: Get addresses from env vars
+  const mintsSet = new Set(
+    config.SPL_TOKEN_MINTS ? config.SPL_TOKEN_MINTS.split(',') : [],
+  )
+  const accountsSet = new Set(
+    config.SPL_TOKEN_ACCOUNTS ? config.SPL_TOKEN_ACCOUNTS.split(',') : [],
+  )
+
+  // @note: Get addresses from custom discovery scripts under "discoveryPath"
+
+  const discoveryPath = path.resolve(
+    config.SPL_TOKEN_DISCOVERY_FOLDER || 'discovery',
+  )
+
+  if (discoveryPath) {
+    try {
+      const files = await new Promise<string[]>((resolve, reject) =>
+        fs.readdir(discoveryPath, (error, files) =>
+          error ? reject(error) : resolve(files),
+        ),
+      )
+
+      const discoveryWhitelist = config.SPL_TOKEN_DISCOVERY_WHITELIST
+        ? config.SPL_TOKEN_DISCOVERY_WHITELIST.split(',')
+        : []
+
+      const filteredFilePaths = []
+
+      for (const file of files) {
+        if (!file.endsWith('.js')) continue
+
+        let filePath = path.join(discoveryPath, file)
+
+        console.log('[Discovery] => whitelisting', file, filePath)
+
+        const stats = await new Promise<Stats>((resolve, reject) =>
+          fs.lstat(filePath, (error, res) =>
+            error ? reject(error) : resolve(res),
+          ),
+        )
+
+        const isDir = stats.isDirectory()
+        const fileName = isDir ? file : file.split('.')[0]
+        filePath = isDir ? path.join(filePath, 'index.js') : filePath
+
+        if (discoveryWhitelist.length && !discoveryWhitelist.includes(fileName))
+          continue
+
+        console.log('[Discovery] => whitelisted', fileName, filePath)
+        filteredFilePaths.push({ fileName, filePath })
+      }
+
+      if (filteredFilePaths.length) {
+        await Promise.all(
+          filteredFilePaths.map(async ({ fileName, filePath }) => {
+            try {
+              const module = await import(filePath)
+              const discoverFn: DiscoveryFn = module.default
+
+              const { accounts, mints } = await discoverFn()
+
+              const countMintBefore = mintsSet.size
+              const countAccountBefore = accountsSet.size
+
+              mints.forEach((mint) => mintsSet.add(mint))
+              accounts.forEach((account) => accountsSet.add(account))
+
+              const countMint = mintsSet.size - countMintBefore
+              const countAccount = accountsSet.size - countAccountBefore
+              const total = countMint + countAccount
+
+              console.log(
+                `[Discovery] => Added ${total} addresses [${countMint} mints] [${countAccount} accounts] from ${fileName}`,
+              )
+            } catch (e) {
+              console.log(`[Discovery] => Error loading file ${fileName}`, e)
+            }
+          }),
+        )
+      }
+    } catch (e) {
+      console.log(`[Discovery] => Error path not found: ${discoveryPath}`, e)
+    }
+  }
+
+  return {
+    mints: [...mintsSet],
+    accounts: [...accountsSet],
+  }
+}

packages/token-snapshot/src/utils/utils.ts

@@ -0,0 +1,146 @@
+import {
+  AlephParsedInnerInstruction,
+  AlephParsedInstruction,
+  AlephParsedParsedInstruction,
+  RawInstruction,
+} from '@aleph-indexer/core'
+import { TOKEN_PROGRAM_ID } from '../constants.js'
+import {
+  SPLTokenRawEvent,
+  SPLTokenEvent,
+  SPLTokenEventType,
+  SPLTokenIncompleteEvent,
+} from '../types.js'
+import BN from 'bn.js'
+
+export function isSPLTokenInstruction(
+  ix: RawInstruction | AlephParsedInstruction | AlephParsedInnerInstruction,
+): ix is SPLTokenRawEvent {
+  return ix.programId === TOKEN_PROGRAM_ID
+}
+
+export function isParsedIx(
+  ix: RawInstruction | AlephParsedInstruction | AlephParsedInnerInstruction,
+): ix is AlephParsedParsedInstruction {
+  return 'parsed' in ix
+}
+
+export function isSPLTokenParsedInstruction(
+  ix: RawInstruction | AlephParsedInstruction | AlephParsedInnerInstruction,
+): ix is SPLTokenRawEvent {
+  if (!isParsedIx(ix) || !isSPLTokenInstruction(ix)) return false
+  return true
+}
+
+export function isSPLTokenMintInstruction(
+  ix: RawInstruction | AlephParsedInstruction | AlephParsedInnerInstruction,
+  mint: string,
+): ix is SPLTokenRawEvent {
+  if (!isSPLTokenParsedInstruction(ix)) return false
+  return getIxMint(ix) === mint
+}
+
+export function isSPLTokenAccountInstruction(
+  ix: RawInstruction | AlephParsedInstruction | AlephParsedInnerInstruction,
+  account: string,
+): ix is SPLTokenRawEvent {
+  if (!isSPLTokenParsedInstruction(ix)) return false
+  return getIxAccounts(ix).includes(account)
+}
+
+export function getIxMint(ix: SPLTokenRawEvent): string | undefined {
+  switch (ix.parsed.type) {
+    case SPLTokenEventType.MintTo:
+    case SPLTokenEventType.MintToChecked:
+    case SPLTokenEventType.Burn:
+    case SPLTokenEventType.BurnChecked:
+    case SPLTokenEventType.InitializeAccount:
+    case SPLTokenEventType.InitializeAccount2:
+    case SPLTokenEventType.InitializeAccount3:
+    case SPLTokenEventType.TransferChecked:
+    case SPLTokenEventType.ApproveChecked:
+    case SPLTokenEventType.InitializeMint:
+    case SPLTokenEventType.InitializeMint2: {
+      return ix.parsed.info.mint
+    }
+  }
+}
+
+export function getIxAccounts(ix: SPLTokenRawEvent): string[] {
+  switch (ix.parsed.type) {
+    case SPLTokenEventType.MintTo:
+    case SPLTokenEventType.MintToChecked:
+    case SPLTokenEventType.Burn:
+    case SPLTokenEventType.BurnChecked:
+    case SPLTokenEventType.InitializeAccount:
+    case SPLTokenEventType.InitializeAccount2:
+    case SPLTokenEventType.InitializeAccount3:
+    case SPLTokenEventType.SetAuthority:
+    case SPLTokenEventType.SyncNative:
+    case SPLTokenEventType.CloseAccount: {
+      return [ix.parsed.info.account]
+    }
+    case SPLTokenEventType.Transfer:
+    case SPLTokenEventType.TransferChecked: {
+      return [ix.parsed.info.source, ix.parsed.info.destination]
+    }
+    case SPLTokenEventType.Approve:
+    case SPLTokenEventType.ApproveChecked:
+    case SPLTokenEventType.Revoke: {
+      return [ix.parsed.info.source]
+    }
+  }
+
+  return []
+}
+
+export function getEventAccounts(event: SPLTokenIncompleteEvent): string[] {
+  switch (event.type) {
+    case SPLTokenEventType.Transfer: {
+      if (event.toAccount) {
+        return [event.account, event.toAccount]
+      } else {
+        return [event.account]
+      }
+    }
+    default: {
+      return [event.account]
+    }
+  }
+}
+
+export function getBalanceFromEvent(
+  event: SPLTokenIncompleteEvent,
+  account: string,
+): string {
+  switch (event.type) {
+    case SPLTokenEventType.Transfer: {
+      if (event.toAccount === account) {
+        return event.toBalance as string
+      } else {
+        return event.balance
+      }
+    }
+    default: {
+      return event.balance
+    }
+  }
+}
+
+export function getMintAndOwnerFromEvent(
+  event: SPLTokenEvent,
+  account: string,
+): { mint: string; owner?: string } {
+  switch (event.type) {
+    case SPLTokenEventType.Transfer: {
+      if (event.toAccount === account) {
+        return { mint: event.mint, owner: event.toOwner }
+      } else {
+        return { mint: event.mint, owner: event.owner }
+      }
+    }
+    default: {
+      return { mint: event.mint, owner: event.owner }
+    }
+  }
+}

packages/token-snapshot/tsconfig.json

@@ -0,0 +1,16 @@
+{
+    "extends": "../../tsconfig.json",
+    "compilerOptions": {
+        "outDir": "dist"
+    },
+    "exclude": [
+        "node_modules",
+        "dist",
+        "scripts",
+        "tests",
+        "**/*.spec.ts",
+        "**/*.test.ts",
+        "**/__tests__",
+        "**/__mocks__"
+    ]
+}

packages/token-snapshot/types.d.ts

@@ -0,0 +1 @@
+export * from '../../types'