refactor: pr feedback and directory restructuring 👌

Author: puffo

.env.example

@@ -52,4 +52,6 @@ ARWEAVE_GATEWAY_URL=https://arweave.net/
 SOUND_XYZ_KEY=asdf
 
 # Seeds API config
+SEEDS_API_PORT=3005
+SEEDS_API_VERSION=1
 PERMITTED_ADMIN_ADDRESSES=0x1,0x2

.mocharc.json

@@ -0,0 +1,11 @@
+{
+  "bail": true,
+  "exit": true,
+  "extension": ["js", "ts"],
+  "file": ["./test/pretest.ts"],
+  "recursive": true,
+  "require": "ts-node/register",
+  "timeout": "15000",
+  "slow": 1000,
+  "watch-files": ["src/**/*.ts", "test/**/*.ts", "test/**/*.ts"]
+}

README.md

@@ -57,6 +57,11 @@ In production, we recommend running the script once a minute to keep it up to da
 watch -n 60 "yarn start >> ./log" 2>&1
 ```
 
+If you want to allow adjustmends to the [CRDT](/crdt.md) seeds via the Seeds API, run:
+```
+yarn serve-seeds-api
+```
+
 ## Operations
 Sometimes things fail (eg: an offchain API is down). This is fine and expected. Things should continue as expected on the next run of the script. Most NFTs/Tracks/Platforms that cause failure/errors are moved into an error queue and retried a few times after some delay so that they don't block progress for the rest of the indexer.
 
@@ -84,6 +89,12 @@ There are a few design goals for the system:
  - Allow extensions with additional metadata or data transformations being added without requiring a DB rebuild nor re-processing from the start
  - Allow for decentralization and even some consensus without any co-ordination
 
+## Tests
+The Seeds API is covered by end-to-end tests. More test coverage to follow as the codebase matures. To run tests locally:
+```
+yarn test:watch
+```
+
 ## Contributing
 The repo is still early and not hyper-polished or perfectly documented. Contributor guidelines are not ready yet, clear development docs and style/standard expectations are not extremely well documented yet. Interfaces are not well documented yet. Keep this in mind, so if you'd like to contribute:
  - First, reach out on Discord and connect with the team (https://discord.gg/8YS3peb62f) so we can help guide you in contributing

package.json

@@ -12,8 +12,8 @@
     "start-logs": "NODE_EXTRA_CA_CERTS=./db-ssl-certificate.pem ts-node src/index.ts 2>&1 | tee /tmp/debug",
     "cli": "NODE_EXTRA_CA_CERTS=./db-ssl-certificate.pem ts-node src/cli/cli.ts",
     "build": "tsc --skipLibCheck",
-    "test": "source .env && NODE_ENV=test mocha -r ts-node/register test/**/*.ts --timeout 15000",
-    "test:watch": "source .env && NODE_ENV=test mocha -r ts-node/register --watch --watch-files src/**/*.ts,test/**/*.ts test/**/*.ts --timeout 15000",
+    "test": "source .env && NODE_ENV=test mocha",
+    "test:watch": "source .env && NODE_ENV=test mocha --watch",
     "bootstrap-db": "NODE_EXTRA_CA_CERTS=./db-ssl-certificate.pem ts-node db/bootstrap.ts",
     "serve": "source .env && DEBUG=graphile-build-pg NODE_EXTRA_CA_CERTS=./db-ssl-certificate.pem npx ts-node ./src/serve.ts --cors",
     "serve-seeds-api": "source .env && NODE_EXTRA_CA_CERTS=./db-ssl-certificate.pem npx ts-node ./src/serve_seeds_api.ts",

src/utils/seedsMiddleware.ts src/seeds/middleware.ts

@@ -0,0 +1,32 @@
+import '../types/env';
+import cors from 'cors';
+import express from 'express';
+
+import { authMiddleware } from './middleware';
+import { persistSeed, validateSeed } from './types';
+
+const apiVersionPrefix = `/v${process.env.SEEDS_API_VERSION || '1'}`;
+
+export const createSeedsAPIServer = () => {
+  const app = express();
+  app.use(express.json());
+  app.use(cors({ origin: true }));
+  app.use(authMiddleware);
+
+  app.post(`${apiVersionPrefix}/seeds/`, async (req, res) => {
+    try {
+      validateSeed(req.body)
+    } catch (e: any) {
+      return res.status(422).send({ error: e.message });
+    }
+
+    try {
+      await persistSeed(req.body)
+      res.sendStatus(200);
+    } catch (e: any) {
+      res.status(500).send({ error: e.message });
+    }
+  });
+
+  return app;
+}

src/seeds/server.ts

@@ -1,12 +1,9 @@
 
 import { Table } from '../db/db';
 import db from '../db/sql-db'
-
-import { ArtistProfileKeys } from './artist';
-import { getCrdtUpdateMessage, getCrdtUpsertMessage } from './message'
-import { NFTContractTypeName, NFTFactoryKeys, NFTStandard } from './nft';
-import { MusicPlatform, MusicPlatformKeys, MusicPlatformType } from './platform';
-import { ProcessedTrackKeys } from './track';
+import { getCrdtUpdateMessage, getCrdtUpsertMessage } from '../types/message'
+import { NFTContractTypeName, NFTStandard } from '../types/nft';
+import { MusicPlatform, MusicPlatformType } from '../types/platform';
 
 enum SeedEntities {
   'platforms',
@@ -21,25 +18,29 @@ enum PlatformsRequiredKeys {
   NAME = 'name',
   TYPE = 'type',
 }
+const PlatformKeys = ['id', 'type', 'name'];
 
 enum NFTFactoriesRequiredKeys {
   ID = 'id',
   PLATFORM_ID = 'platformId',
-  CONTRACT_TYPE = 'contractType', // validated against `NFTContractTypeName`
-  STANDARD = 'standard', // validated against `NFTStandard`
+  CONTRACT_TYPE = 'contractType',
+  STANDARD = 'standard',
   AUTO_APPROVE = 'autoApprove',
   APPROVED = 'approved'
 }
+const NFTFactoryKeys = ['id', 'platformId', 'contractType', 'name', 'symbol', 'typeMetadata', 'standard', 'autoApprove', 'approved'];
 
 enum ArtistProfilesRequiredKeys {
   ARTIST_ID = 'artistId',
   PLATFORM_ID = 'platformId',
 }
+const ArtistProfileKeys = ['platformInternalId', 'artistId', 'name', 'platformId', 'avatarUrl', 'websiteUrl'];
 
 enum ProcessedTracksRequiredKeys {
   ARTIST_ID = 'artistId',
   PLATFORM_ID = 'platformId',
 }
+const ProcessedTrackKeys = ['platformInternalId', 'title', 'slug', 'platformId', 'description', 'websiteUrl', 'artistId', 'lossyAudioURL', 'lossyAudioIPFSHash', 'lossyArtworkURL', 'lossyArtworkIPFSHash'];
 
 type SeedPayload = {
   entity: SeedEntity,
@@ -52,7 +53,7 @@ export const validateSeed = (payload: SeedPayload): void => {
   const validatorFunctions = {
     'platforms': [
       () => minimumKeysPresent(payload, Object.values(PlatformsRequiredKeys)),
-      () => onlyValidKeysPresent(payload, MusicPlatformKeys),
+      () => onlyValidKeysPresent(payload, PlatformKeys),
       () => typeValidator(payload, 'type', MusicPlatformType),
     ],
     'nftFactories': [

src/types/seeds.ts src/seeds/types.ts

@@ -1,37 +1,5 @@
-import 'dotenv/config';
-import './types/env';
-import cors from 'cors';
-import express from 'express';
+import { createSeedsAPIServer } from './seeds/server';
 
-import { persistSeed, validateSeed } from './types/seeds';
-import { authMiddleware } from './utils/seedsMiddleware';
-
-const apiVersionPrefix = '/v1';
-
-const app = express();
-app.use(express.json());
-app.use(cors({ origin: true }));
-app.use(authMiddleware);
-
-app.post(`${apiVersionPrefix}/seeds/`, async (req, res) => {
-  try {
-    validateSeed(req.body)
-  } catch (e: any) {
-    return res.status(422).send({ error: e.message });
-  }
-
-  try {
-    await persistSeed(req.body)
-    res.sendStatus(200);
-  } catch (e: any) {
-    res.status(500).send({ error: e.message });
-  }
+createSeedsAPIServer().listen(process.env.SEEDS_API_PORT, () => {
+  console.log(`Server running on port ${process.env.SEEDS_API_PORT}`);
 });
-
-if (process.env.NODE_ENV !== 'test') {
-  app.listen(3005, () => {
-    console.log(`Server running on port ${3005}`);
-  });
-}
-
-module.exports = app;

src/serve_seeds_api.ts

@@ -16,8 +16,6 @@ export type ArtistProfile = TimeField & {
   websiteUrl?: string;
 }
 
-export const ArtistProfileKeys = ['platformInternalId', 'artistId', 'name', 'platformId', 'avatarUrl', 'websiteUrl'];
-
 export const mapArtist = (artistProfile: ArtistProfile): Artist => {
   return {
     name: artistProfile.name,

src/types/artist.ts

@@ -69,7 +69,6 @@ export type NftFactory = Contract & {
   autoApprove: boolean; // automaticlly approve nfts generated by this facotry
   approved: boolean; // should index nfts generated by this factory
 }
-export const NFTFactoryKeys = ['id', 'platformId', 'contractType', 'name', 'symbol', 'typeMetadata', 'standard', 'autoApprove', 'approved'];
 
 export type NFTContractType = {
   contractCalls: ValidContractNFTCallFunction[],

src/types/nft.ts

@@ -31,7 +31,6 @@ export type MusicPlatform = {
   type: MusicPlatformType
   name: string
 }
-export const MusicPlatformKeys: Array<keyof MusicPlatform> = ['id', 'type', 'name'];
 
 export type PlatformMapper = {
   mapNFTsToTrackIds: MapNFTsToTrackIds

src/types/platform.ts

@@ -26,8 +26,6 @@ export type ProcessedTrack = Record & {
   artistId: string;
 } & ProcessedTrackAudio & ProcessedTrackArtwork;
 
-export const ProcessedTrackKeys = ['platformInternalId', 'title', 'slug', 'platformId', 'description', 'websiteUrl', 'artistId', 'lossyAudioURL', 'lossyAudioIPFSHash', 'lossyArtworkURL', 'lossyArtworkIPFSHash'];
-
 export const mergeProcessedTracks = async (newProcessedTracks: ProcessedTrack[], dbClient: DBClient, prioritizeNew: boolean) => {
   const platformInternalIds = newProcessedTracks.map(t => t.platformInternalId);
   const existingProcessedTracks = await dbClient.getRecords<ProcessedTrack>(Table.processedTracks,

src/types/track.ts

@@ -0,0 +1,13 @@
+export const TEST_ADMIN_WALLET = {
+  address: '0x8eb97c37B0BDe7A09eA5b49D6D97cd57e10559ba',
+  privateKey: '0xe07cc69757e3b261ffeb70df20f832ae74da57e11dd440a5da75377abe8caefc',
+}
+
+console.log('-------------------------------------');
+console.log('configuring tests...');
+
+process.env.PERMITTED_ADMIN_ADDRESSES = TEST_ADMIN_WALLET.address;
+
+console.log(`setting PERMITTED_ADMIN_ADDRESSES...`);
+console.log('starting tests...');
+console.log('-------------------------------------');

test/pretest.ts

@@ -1,24 +1,18 @@
 import supertest from 'supertest'
 import web3 from 'web3';
 
-import { MusicPlatformType } from '../src/types/platform';
+import { createSeedsAPIServer } from '../../src/seeds/server';
+import { TEST_ADMIN_WALLET } from '../pretest';
 
-describe('Seeds API', () => {
+const throwDBHint = (err: any) => { throw new Error(`${err.message}\nHINT: tests run against dev DB. Ensure that DB running, migrated, and working as expected`) };
+
+describe('Seeds API server', () => {
   let app: any;
   const Web3 = new web3();
+  const wallet = Web3.eth.accounts.privateKeyToAccount(TEST_ADMIN_WALLET.privateKey)
 
-  // TODO: rather stub out the permittedAdminAddresses() to contain this address
-  // instead of having to put the known address into .env
-  const testAdminWallet = {
-    address: '0x8eb97c37B0BDe7A09eA5b49D6D97cd57e10559ba',
-    privateKey: '0xe07cc69757e3b261ffeb70df20f832ae74da57e11dd440a5da75377abe8caefc',
-  }
-
-  const wallet = Web3.eth.accounts.privateKeyToAccount(testAdminWallet.privateKey)
-
-  // TODO: spin up a test DB instead of using development
   before(() => {
-    app = require('../src/serve_seeds_api.ts');
+    app = createSeedsAPIServer();
   });
 
   describe('un-authenticated', () => {
@@ -37,7 +31,6 @@ describe('Seeds API', () => {
         .set('x-signature', signature)
         .expect(403)
         .end((err,res) => { if (err) throw err });
-
     })
   })
 
@@ -53,7 +46,6 @@ describe('Seeds API', () => {
           .set('x-signature', signature)
           .expect(422, { error: 'unknown seed entity' })
           .end((err,res) => { if (err) throw err });
-
       })
     })
 
@@ -84,7 +76,7 @@ describe('Seeds API', () => {
 
       describe('with unsupported fields', () => {
         it('returns an error', () => {
-          const body = { entity: 'platforms', data: { id: 'potato', name: 'potato', type: MusicPlatformType.sound, hackyou: 'boo' } }
+          const body = { entity: 'platforms', data: { id: 'potato', name: 'potato', type: 'sound', hackyou: 'boo' } }
           const signature = wallet.sign(JSON.stringify(body)).signature
 
           supertest(app).post(endpoint).send(body)
@@ -96,13 +88,13 @@ describe('Seeds API', () => {
 
       describe('with a valid payload', () => {
         it('returns a 200', () => {
-          const body = { entity: 'platforms', data: { id: 'jamboni', name: 'Jamboni Jams', type: MusicPlatformType.sound } }
+          const body = { entity: 'platforms', data: { id: 'jamboni', name: 'Jamboni Jams', type: 'sound' } }
           const signature = wallet.sign(JSON.stringify(body)).signature;
 
           supertest(app).post(endpoint).send(body)
             .set('x-signature', signature)
             .expect(200)
-            .end((err,res) => { if (err) throw err });
+            .end((err,res) => { if (err) { throwDBHint(err) } });
         })
         it('persists the seed');
       })
@@ -177,7 +169,7 @@ describe('Seeds API', () => {
           supertest(app).post(endpoint).send(body)
             .set('x-signature', signature)
             .expect(200)
-            .end((err,res) => { if (err) throw err });
+            .end((err,res) => { if (err) throwDBHint(err) });
         })
         it('persists the seed');
       })
@@ -222,7 +214,7 @@ describe('Seeds API', () => {
           supertest(app).post(endpoint).send(body)
             .set('x-signature', signature)
             .expect(200)
-            .end((err,res) => { if (err) throw err });
+            .end((err,res) => { if (err) throwDBHint(err) });
         })
         it('persists the seed');
       })
@@ -267,7 +259,7 @@ describe('Seeds API', () => {
           supertest(app).post(endpoint).send(body)
             .set('x-signature', signature)
             .expect(200)
-            .end((err,res) => { if (err) { throw err } });
+            .end((err,res) => { if (err) { throwDBHint(err) } });
         })
         it('persists the seed');
       })