Added fastq manager

config/config.ts

@@ -67,6 +67,7 @@ import { getDataMigrateStackProps } from './stacks/dataMigrate';
 import { getHtsgetProps } from './stacks/htsget';
 import { getSampleSheetCheckerProps } from './stacks/sampleSheetChecker';
 import { getAccessKeySecretStackProps } from './stacks/accessKeySecret';
+import { getFastqManagerStackProps, getFastqManagerTableStackProps } from './stacks/fastqManager';
 
 interface EnvironmentConfig {
   name: string;
@@ -108,6 +109,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
       oncoanalyserPipelineTableStackProps: getOncoanalyserPipelineTableStackProps(),
       sashPipelineTableStackProps: getSashPipelineTableStackProps(),
       accessKeySecretStackProps: getAccessKeySecretStackProps(stage),
+      fastqManagerTableStackProps: getFastqManagerTableStackProps(stage),
     },
     statelessConfig: {
       metadataManagerStackProps: getMetadataManagerStackProps(stage),
@@ -138,6 +140,7 @@ export const getEnvironmentConfig = (stage: AppStage): EnvironmentConfig | null
       htsgetProps: getHtsgetProps(stage),
       sampleSheetCheckerProps: getSampleSheetCheckerProps(stage),
       pgDDProps: getPgDDProps(stage),
+      fastqManagerStackProps: getFastqManagerStackProps(stage),
     },
   };
 

config/constants.ts

@@ -88,6 +88,11 @@ export const icav2PipelineCacheBucket: Record<AppStage, string> = {
   [AppStage.GAMMA]: 'pipeline-stg-cache-503977275616-ap-southeast-2',
   [AppStage.PROD]: 'pipeline-prod-cache-503977275616-ap-southeast-2',
 };
+export const icav2PipelineCachePrefix: Record<AppStage, string> = {
+  [AppStage.BETA]: 'byob-icav2/development/',
+  [AppStage.GAMMA]: 'byob-icav2/staging/',
+  [AppStage.PROD]: 'byob-icav2/production/',
+};
 
 // The test inventory bucket for dev.
 export const fileManagerInventoryBucket: Record<AppStage.BETA, string> = {
@@ -892,3 +897,42 @@ export const oraDecompressionIcav2ReadyEventSource = 'orcabus.workflowmanager';
 export const oraDecompressionIcav2EventSource = 'orcabus.oradecompression';
 export const oraDecompressionIcav2EventDetailType = 'FastqListRowDecompressed';
 export const oraDecompressionStateMachinePrefix = 'oraDecompressionSfn';
+
+/*
+Fastq Manager
+*/
+
+// Tables
+export const fastqListRowTableName = 'fastqManagerDynamoDBTable';
+export const fastqSetTableName = 'fastqSetDynamoDBTable';
+export const fastqJobTableName = 'fastqJobDynamoDBTable';
+
+// Table indexes
+export const fastqListRowManagerIndexes = [
+  'rgid_ext',
+  'instrument_run_id',
+  'library_orcabus_id',
+  'fastq_set_id',
+];
+export const fastqSetManagerIndexes = ['rgid_ext', 'instrument_run_id', 'library_orcabus_id'];
+export const fastqJobManagerIndexes = ['fastq_id', 'job_type', 'status'];
+
+// S3 Buckets
+export const fastqManagerCacheBucket: Record<AppStage, string> = {
+  [AppStage.BETA]: `fastq-manager-cache-${accountIdAlias.beta}-ap-southeast-2`,
+  [AppStage.GAMMA]: `fastq-manager-cache-${accountIdAlias.gamma}-ap-southeast-2`,
+  [AppStage.PROD]: `fastq-manager-cache-${accountIdAlias.prod}-ap-southeast-2`,
+};
+
+export const ntsmBucket: Record<AppStage, string> = {
+  [AppStage.BETA]: `ntsm-fingerprints-${accountIdAlias.beta}-ap-southeast-2`,
+  [AppStage.GAMMA]: `ntsm-fingerprints-${accountIdAlias.gamma}-ap-southeast-2`,
+  [AppStage.PROD]: `ntsm-fingerprints-${accountIdAlias.prod}-ap-southeast-2`,
+};
+
+// Events
+export const fastqManagerEventSource = 'orcabus.fastqmanager';
+export const fastqManagerEventDetails = {
+  updateFastqListRow: 'FastqListRowStateChange',
+  updateFastqSet: 'FastqSetStateChange',
+};

config/stacks/fastqManager.ts

@@ -0,0 +1,84 @@
+import {
+  AppStage,
+  fastqListRowTableName,
+  cognitoApiGatewayConfig,
+  corsAllowOrigins,
+  logsApiGatewayConfig,
+  jwtSecretName,
+  hostedZoneNameParameterPath,
+  fastqListRowManagerIndexes,
+  fastqSetTableName,
+  fastqSetManagerIndexes,
+  fastqJobTableName,
+  fastqJobManagerIndexes,
+  fastqManagerCacheBucket,
+  ntsmBucket,
+  fastqManagerEventSource,
+  fastqManagerEventDetails,
+  icav2PipelineCacheBucket,
+  icav2PipelineCachePrefix,
+  eventBusName,
+} from '../constants';
+import { FastqManagerTableConfig } from '../../lib/workload/stateful/stacks/fastq-manager-db/deploy/stack';
+import { FastqManagerStackConfig } from '../../lib/workload/stateless/stacks/fastq-manager/deploy/interfaces';
+
+// Stateful
+export const getFastqManagerTableStackProps = (stage: AppStage): FastqManagerTableConfig => {
+  return {
+    /* DynamoDB table for fastq list rows */
+    fastqListRowDynamodbTableName: fastqListRowTableName,
+    fastqSetDynamodbTableName: fastqSetTableName,
+    fastqJobDynamodbTableName: fastqJobTableName,
+    /* Buckets */
+    fastqManagerCacheBucketName: fastqManagerCacheBucket[stage],
+    ntsmBucketName: ntsmBucket[stage],
+  };
+};
+
+// Stateless
+export const getFastqManagerStackProps = (stage: AppStage): FastqManagerStackConfig => {
+  return {
+    /*
+    API Gateway props
+    */
+    apiGatewayCognitoProps: {
+      ...cognitoApiGatewayConfig,
+      corsAllowOrigins: corsAllowOrigins[stage],
+      apiGwLogsConfig: logsApiGatewayConfig[stage],
+      apiName: 'FastqManager',
+      customDomainNamePrefix: 'fastq',
+    },
+
+    /*
+    Orcabus token and zone name for external lambda functions
+    */
+    orcabusTokenSecretsManagerPath: jwtSecretName,
+    hostedZoneNameSsmParameterPath: hostedZoneNameParameterPath,
+
+    /*
+    Data tables
+    */
+    fastqListRowDynamodbTableName: fastqListRowTableName,
+    fastqSetDynamodbTableName: fastqSetTableName,
+    fastqJobsDynamodbTableName: fastqJobTableName,
+    /* Indexes - need permissions to query indexes */
+    fastqListRowDynamodbIndexes: fastqListRowManagerIndexes,
+    fastqSetDynamodbIndexes: fastqSetManagerIndexes,
+    fastqJobsDynamodbIndexes: fastqJobManagerIndexes,
+
+    /*
+    Buckets stuff
+    */
+    pipelineCacheBucketName: icav2PipelineCacheBucket[stage],
+    pipelineCachePrefix: icav2PipelineCachePrefix[stage],
+    fastqManagerCacheBucketName: fastqManagerCacheBucket[stage],
+    ntsmBucketName: ntsmBucket[stage],
+
+    /*
+    Event bus stuff
+    */
+    eventBusName: eventBusName,
+    eventSource: fastqManagerEventSource,
+    eventDetailType: fastqManagerEventDetails,
+  };
+};

config/stacks/fileManager.ts

@@ -18,6 +18,7 @@ import {
   oncoanalyserBucket,
   region,
   vpcProps,
+  ntsmBucket,
 } from '../constants';
 
 export const fileManagerBuckets = (stage: AppStage): string[] => {
@@ -27,6 +28,8 @@ export const fileManagerBuckets = (stage: AppStage): string[] => {
     eventSourceBuckets.push(icav2ArchiveAnalysisBucket[stage]);
     eventSourceBuckets.push(icav2ArchiveFastqBucket[stage]);
   }
+  eventSourceBuckets.push(ntsmBucket[stage]);
+
   return eventSourceBuckets;
 };
 

config/stacks/shared.ts

@@ -17,6 +17,7 @@ import {
   icav2ArchiveAnalysisBucket,
   icav2ArchiveFastqBucket,
   icav2PipelineCacheBucket,
+  ntsmBucket,
   oncoanalyserBucket,
   rdsMasterSecretName,
   vpcProps,
@@ -160,6 +161,13 @@ export const getEventSourceConstructProps = (stage: AppStage): EventSourceProps
     });
   }
 
+  // Add the ntsm bucket rule
+  props.rules.push({
+    bucket: ntsmBucket[stage],
+    eventTypes,
+    patterns: eventSourcePattern(),
+  });
+
   return props;
 };
 

lib/workload/stateful/stacks/fastq-manager-db/Readme.md

@@ -0,0 +1,22 @@
+# Fastq Manager DB
+
+This comprises three dynamodb table that stores the metadata of fastq files. 
+
+The tables are:
+* fastq list row
+* fastq set
+* jobs
+
+The fastq list row table is indexed by the id with the following global secondary indexes:
+  * rgid_ext (the read group id, plus the instrument run id)
+  * instrument_run_id (the instrument run id)
+  * library_id (the library orcabus id)
+  * fastq_set_id (easier to find the corresponding set object)
+
+The fastq set table is indexed by the id with the following global secondary indexes:
+  * library_id (the library orcabus id)
+
+The jobs table is indexed by the id with the following global secondary indexes:
+  * fastq_set_id (the fastq set id)
+  * status (the status of the job)
+