Fastq manager updates

Introduced 'ResponseDict' concept to separate Response Classes from Response Dictionaries, model_dump() -> Self isn't always a viable solution. This also allows us to have dynamic logic in our file objects, either with s3 details or without s3 details.

* NTSM Evaluation fixes

Author: alexiswl

Diff for: lib/workload/stateful/stacks/fastq-manager-db/deploy/stack.ts

@@ -196,11 +196,14 @@ export class FastqManagerTable extends cdk.Stack {
   }
 
   private add_buckets(props: FastqManagerTableStackProps) {
+    // Add the ntsm bucket with event bridge enabled
     new s3.Bucket(this, 'ntsm_bucket', {
       bucketName: props.ntsmBucketName,
       removalPolicy: props.removalPolicy || RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE,
+      eventBridgeEnabled: true, // So that the filemanager can listen to events
     });
 
+    // Add fastq manager cache bucket
     new s3.Bucket(this, 'fastq_manager_cache_bucket', {
       bucketName: props.fastqManagerCacheBucketName,
       removalPolicy: props.removalPolicy || RemovalPolicy.RETAIN_ON_UPDATE_OR_DELETE,

Diff for: lib/workload/stateless/stacks/fastq-manager/app/api/fastq_manager_api_tools/api/v1/routers/__init__.py

@@ -226,8 +226,8 @@ def run_ntsm_eval(fastq_set_id_x: str, fastq_set_id_y: Optional[str] = None) ->
     else:
         env_var = RUN_NTSM_EVAL_X_Y_AWS_STEP_FUNCTION_ARN_ENV_VAR
         input_dict = {
-            "fastqSetIdX": fastq_set_id_x,
-            "fastqSetIdY": fastq_set_id_y
+            "fastqSetIdA": fastq_set_id_x,
+            "fastqSetIdB": fastq_set_id_y
         }
 
     # Run qc stats through the AWS step function

Diff for: lib/workload/stateless/stacks/fastq-manager/app/api/fastq_manager_api_tools/api/v1/routers/fastq_list_row.py

@@ -51,8 +51,10 @@
 
 # Model imports
 from ....models import BoolQueryEnum, FastqListRowDict, PresignedUrlModel, QueryPagination
-from ....models.fastq_list_row import FastqListRowResponse, FastqListRowData, FastqListRowCreate, \
-    FastqListRowListResponse, FastqListRowQueryPaginatedResponse
+from ....models.fastq_list_row import (
+    FastqListRowResponse, FastqListRowData, FastqListRowCreate,
+    FastqListRowListResponse, FastqListRowQueryPaginatedResponse, FastqListRowResponseDict
+)
 from ....models.fastq_pair import FastqPairStorageObjectPatch, FastqPairStorageObjectData
 from ....models.fastq_set import FastqSetData
 from ....models.file_compression_info import FileCompressionInfoPatch, FileCompressionInfoData
@@ -334,7 +336,7 @@ async def list_fastq(
                     "includeS3Details": include_s3_details,
                 },
                 **pagination
-            )
+            ).items()
         )),
     )
 
@@ -353,9 +355,11 @@ async def get_fastq(
             alias="includeS3Details",
             description="Include the s3 details such as s3 uri and storage class"
         ),
-) -> FastqListRowResponse:
+) -> FastqListRowResponseDict:
     try:
-        return FastqListRowData.get(fastq_id).to_dict(include_s3_details=include_s3_details)
+        return FastqListRowData.get(fastq_id).to_dict(
+            include_s3_details=include_s3_details
+        )
     except DoesNotExist as e:
         raise HTTPException(status_code=404, detail=str(e))
 
@@ -369,7 +373,7 @@ async def get_fastq(
     Please use the fastqSet endpoint if registering multiple fastqs simultaneously to reduce race conditions
     """)
 )
-async def create_fastq(fastq_obj: FastqListRowCreate) -> FastqListRowResponse:
+async def create_fastq(fastq_obj: FastqListRowCreate) -> FastqListRowResponseDict:
     # First convert the CreateFastqListRow to a FastqListRow
     fastq_obj = FastqListRowData(**dict(fastq_obj.model_dump(by_alias=True)))
 
@@ -411,7 +415,7 @@ async def get_fastq_list_row(fastq_id: str = Depends(sanitise_fqr_orcabus_id)) -
     description="Update the library associated with a Fastq List Row Object"
 )
 async def update_library(fastq_id: str = Depends(sanitise_fqr_orcabus_id),
-                         library_obj: LibraryPatch = Depends()) -> FastqListRowResponse:
+                         library_obj: LibraryPatch = Depends()) -> FastqListRowResponseDict:
     fastq_obj = FastqListRowData.get(fastq_id)
 
     library_obj = LibraryData(**dict(library_obj.model_dump(by_alias=True)))
@@ -506,7 +510,7 @@ async def get_jobs(
     description="Add QC Stats to a Fastq List Row Object"
 )
 async def add_qc_stats(fastq_id: str = Depends(sanitise_fqr_orcabus_id),
-                       qc_obj: QcInformationPatch = Depends()) -> FastqListRowResponse:
+                       qc_obj: QcInformationPatch = Depends()) -> FastqListRowResponseDict:
     fastq_obj = FastqListRowData.get(fastq_id)
     fastq_obj.qc = QcInformationData(**dict(qc_obj.model_dump(by_alias=True)))
     fastq_obj.save()
@@ -527,7 +531,7 @@ async def add_qc_stats(fastq_id: str = Depends(sanitise_fqr_orcabus_id),
     description="Add Read Count Information to a Fastq List Row Object"
 )
 async def add_read_count(fastq_id: str = Depends(sanitise_fqr_orcabus_id),
-                         read_count_obj: ReadCountInfoPatch = Depends()) -> FastqListRowResponse:
+                         read_count_obj: ReadCountInfoPatch = Depends()) -> FastqListRowResponseDict:
     fastq = FastqListRowData.get(fastq_id)
 
     # Get read count info
@@ -553,7 +557,7 @@ async def add_read_count(fastq_id: str = Depends(sanitise_fqr_orcabus_id),
     description="Add File Compression Information to a Fastq List Row Object"
 )
 async def add_file_compression(fastq_id: str = Depends(sanitise_fqr_orcabus_id),
-                               file_compression_obj: FileCompressionInfoPatch = Depends()) -> FastqListRowResponse:
+                               file_compression_obj: FileCompressionInfoPatch = Depends()) -> FastqListRowResponseDict:
     # Get fastq object
     fastq = FastqListRowData.get(fastq_id)
 
@@ -591,7 +595,7 @@ async def add_file_compression(fastq_id: str = Depends(sanitise_fqr_orcabus_id),
     description="Add Ntsm Storage Object to a Fastq List Row Object"
 )
 async def add_ntsm_uri(fastq_id: str = Depends(sanitise_fqr_orcabus_id),
-                       ntsm: NtsmUriUpdate = Depends()) -> FastqListRowResponse:
+                       ntsm: NtsmUriUpdate = Depends()) -> FastqListRowResponseDict:
     fastq = FastqListRowData.get(fastq_id)
     fastq.ntsm = NtsmUriData(**dict(ntsm.model_dump())).ntsm
     fastq.save()
@@ -612,7 +616,7 @@ async def add_ntsm_uri(fastq_id: str = Depends(sanitise_fqr_orcabus_id),
     tags=["fastq validate"],
     description="Validate a Fastq List Row Object"
 )
-async def validate_fastq(fastq_id: str = Depends(sanitise_fqr_orcabus_id)) -> FastqListRowResponse:
+async def validate_fastq(fastq_id: str = Depends(sanitise_fqr_orcabus_id)) -> FastqListRowResponseDict:
     fastq = FastqListRowData.get(fastq_id)
     fastq.is_valid = True
     fastq.save()
@@ -632,7 +636,7 @@ async def validate_fastq(fastq_id: str = Depends(sanitise_fqr_orcabus_id)) -> Fa
     tags=["fastq validate"],
     description="Invalidate a Fastq List Row Object, this is useful if an instrument run has failed"
 )
-async def invalidate_fastq(fastq_id: str = Depends(sanitise_fqr_orcabus_id)) -> FastqListRowResponse:
+async def invalidate_fastq(fastq_id: str = Depends(sanitise_fqr_orcabus_id)) -> FastqListRowResponseDict:
     fastq_obj = FastqListRowData.get(fastq_id)
 
     # Check if the fastq is part of a fastq set
@@ -659,7 +663,7 @@ async def invalidate_fastq(fastq_id: str = Depends(sanitise_fqr_orcabus_id)) ->
     description="Add Fastq Pair Storage Object to a Fastq List Row Object"
 )
 async def add_fastq_pair_storage_object(fastq_id: str = Depends(sanitise_fqr_orcabus_id),
-                                        fastq_pair_storage_obj: FastqPairStorageObjectPatch = Depends()) -> FastqListRowResponse:
+                                        fastq_pair_storage_obj: FastqPairStorageObjectPatch = Depends()) -> FastqListRowResponseDict:
     fastq = FastqListRowData.get(fastq_id)
     # Check that no fastqPairStorageObject exists for this fastq id
     try:
@@ -682,7 +686,7 @@ async def add_fastq_pair_storage_object(fastq_id: str = Depends(sanitise_fqr_orc
     tags=["fastq update"],
     description="Remove Fastq Pair Storage Object from a Fastq List Row Object"
 )
-async def remove_fastq_pair_storage_object(fastq_id: str = Depends(sanitise_fqr_orcabus_id)) -> FastqListRowResponse:
+async def remove_fastq_pair_storage_object(fastq_id: str = Depends(sanitise_fqr_orcabus_id)) -> FastqListRowResponseDict:
     fastq = FastqListRowData.get(fastq_id)
     # Check that the fastqPairStorageObject exists for this fastq id
     try:

Diff for: lib/workload/stateless/stacks/fastq-manager/app/api/fastq_manager_api_tools/api/v1/routers/fastq_set.py

@@ -45,7 +45,7 @@
 from ....models.fastq_list_row import FastqListRowData
 from ....models.fastq_set import (
     FastqSetData, FastqSetResponse, FastqSetListResponse, FastqSetCreate,
-    FastqSetQueryPaginatedResponse
+    FastqSetQueryPaginatedResponse, FastqSetResponseDict
 )
 from ....models.library import LibraryData
 from ....models.merge_fastq_sets import MergePatch
@@ -54,6 +54,8 @@
     is_orcabus_ulid,
     sanitise_fqs_orcabus_id,
     sanitise_fqr_orcabus_id,
+    sanitise_fqs_orcabus_id_x,
+    sanitise_fqs_orcabus_id_y
 )
 
 router = APIRouter()
@@ -217,7 +219,7 @@ async def list_fastq_sets(
                     lambda fqs_iter_: FastqSetData.from_response(**fqs_iter_.to_dict()),
                     query_lists[0]
                 )),
-                include_s3_detail=include_s3_details
+                include_s3_details=include_s3_details
             ).model_dump(),
             query_pagination=pagination,
             params_response=dict(filter(
@@ -253,7 +255,7 @@ async def list_fastq_sets(
                     query_lists[0]
                 )
             )),
-            include_s3_detail=include_s3_details
+            include_s3_details=include_s3_details
         ).model_dump(by_alias=True),
         query_pagination=pagination,
         params_response=dict(filter(
@@ -280,27 +282,38 @@ async def list_fastq_sets(
     Please use the fastqSet endpoint if registering multiple fastqs simultaneously to reduce race conditions
     """)
 )
-async def create_fastq(fastq_set_obj_create: FastqSetCreate) -> FastqSetResponse:
+async def create_fastq(fastq_set_obj_create: FastqSetCreate) -> FastqSetResponseDict:
     # Confirm that the library id matches those in the fastq objects
     if len(set(list(map(
         lambda fastq_obj: fastq_obj.library.orcabus_id,
-        fastq_set_obj_create.fastq_set
+        # Iterate over each object in the fastq set, 'get' object if of type 'string'
+        # Otherwise parse the object itself
+        list(map(
+            lambda fastq_obj_iter_: (
+                    FastqListRowData.get(fastq_obj_iter_)
+                    if isinstance(fastq_obj_iter_, str)
+                    else fastq_obj_iter_
+            ),
+            fastq_set_obj_create.fastq_set
+        ))
     )))) > 1:
         raise HTTPException(
             status_code=409,
             detail="Got multiple different library ids in the fastq objects"
         )
 
     # Confirm that the library id matches the library id in the fastq set
-    if fastq_set_obj_create.library.orcabus_id != fastq_set_obj_create.fastq_set[0].library.orcabus_id:
-        raise HTTPException(
-            status_code=409,
-            detail="Fastq set library id does not match those of the fastq objects"
-        )
+    first_fastq_obj = fastq_set_obj_create.fastq_set[0]
+    if isinstance(first_fastq_obj, str):
+        first_fastq_obj = FastqListRowData.get(first_fastq_obj)
 
     # Confirm that all fastq sets are unique
     rgid_exts = list(set(list(map(
-        lambda fastq_obj_iter_: FastqListRowData(**dict(fastq_obj_iter_.model_dump(by_alias=True))).rgid_ext,
+        lambda fastq_obj_iter_: (
+            FastqListRowData.get(fastq_obj_iter_).rgid_ext
+            if isinstance(fastq_obj_iter_, str)
+            else FastqListRowData(**dict(fastq_obj_iter_.model_dump(by_alias=True))).rgid_ext
+        ),
         fastq_set_obj_create.fastq_set
     ))))
     if len(rgid_exts) != len(fastq_set_obj_create.fastq_set):
@@ -398,6 +411,13 @@ async def create_fastq(fastq_set_obj_create: FastqSetCreate) -> FastqSetResponse
             detail="Fastq set contains invalid fastqs"
         )
 
+    # Ensure that the library id matches the library id in the fastq set
+    if fastq_set_data_obj.library.orcabus_id != first_fastq_obj.library.orcabus_id:
+        raise HTTPException(
+            status_code=409,
+            detail="Fastq set library id does not match those of the fastq objects"
+        )
+
     # Add the fastq_set_id to the fastq objects
     for fastq_obj in fastq_data_objs:
         fastq_obj.fastq_set_id = fastq_set_data_obj.id
@@ -430,18 +450,18 @@ async def create_fastq(fastq_set_obj_create: FastqSetCreate) -> FastqSetResponse
 async def get_fastq(
         fastq_set_id: str = Depends(sanitise_fqs_orcabus_id),
         # Include s3 uri - resolve the s3 uri if requested
-        include_s3_detail: Optional[bool] = Query(
+        include_s3_details: Optional[bool] = Query(
             default=False,
-            alias="includeS3Detail",
+            alias="includeS3Details",
             description="Include the s3 uris for the fastq objects"
         )
-) -> FastqSetResponse:
+) -> FastqSetResponseDict:
     try:
         return FastqSetListResponse(
             fastq_set_list=[
                 FastqSetData.get(fastq_set_id)
             ],
-            include_s3_detail=include_s3_detail
+            include_s3_details=include_s3_details
         ).model_dump(by_alias=True)[0]
     except DoesNotExist as e:
         raise HTTPException(status_code=404, detail=str(e))
@@ -465,7 +485,7 @@ async def get_fastq_list_row(fastq_set_id: str = Depends(sanitise_fqs_orcabus_id
     tags=["fastqset update"],
     description="Link a fastq object to this fastq set"
 )
-async def link_fastq(fastq_set_id: str = Depends(sanitise_fqs_orcabus_id), fastq_id = Depends(sanitise_fqr_orcabus_id)) -> FastqSetResponse:
+async def link_fastq(fastq_set_id: str = Depends(sanitise_fqs_orcabus_id), fastq_id = Depends(sanitise_fqr_orcabus_id)) -> FastqSetResponseDict:
     fastq_set_obj = FastqSetData.get(fastq_set_id)
     fastq_obj = FastqListRowData.get(fastq_id)
 
@@ -574,7 +594,7 @@ async def unlink_fastq(fastq_set_id: str = Depends(sanitise_fqs_orcabus_id), fas
     tags=["fastqset update"],
     description="Set this fastq set as the current fastq set for this library"
 )
-async def set_current_fastq_set(fastq_set_id: str = Depends(sanitise_fqs_orcabus_id)) -> FastqSetResponse:
+async def set_current_fastq_set(fastq_set_id: str = Depends(sanitise_fqs_orcabus_id)) -> FastqSetResponseDict:
     fastq_set_obj = FastqSetData.get(fastq_set_id)
 
     # Check fastq set exists
@@ -620,7 +640,7 @@ async def set_current_fastq_set(fastq_set_id: str = Depends(sanitise_fqs_orcabus
     tags=["fastqset update"],
     description="Set current fastq set flag to false for this fastq set"
 )
-async def set_current_fastq_set(fastq_set_id: str = Depends(sanitise_fqs_orcabus_id)) -> FastqSetResponse:
+async def set_current_fastq_set(fastq_set_id: str = Depends(sanitise_fqs_orcabus_id)) -> FastqSetResponseDict:
     fastq_set_obj = FastqSetData.get(fastq_set_id)
 
     # Check fastq set exists
@@ -653,7 +673,7 @@ async def set_current_fastq_set(fastq_set_id: str = Depends(sanitise_fqs_orcabus
     tags=["fastqset update"],
     description="Allow additional fastqs to this fastq set"
 )
-async def set_allow_additional_fastqs(fastq_set_id: str = Depends(sanitise_fqs_orcabus_id)) -> FastqSetResponse:
+async def set_allow_additional_fastqs(fastq_set_id: str = Depends(sanitise_fqs_orcabus_id)) -> FastqSetResponseDict:
     fastq_set_obj = FastqSetData.get(fastq_set_id)
 
     # Check fastq set exists
@@ -702,7 +722,7 @@ async def set_allow_additional_fastqs(fastq_set_id: str = Depends(sanitise_fqs_o
     tags=["fastqset update"],
     description="Disallow additional fastqs to this fastq set"
 )
-async def set_allow_additional_fastqs_to_false(fastq_set_id: str = Depends(sanitise_fqs_orcabus_id)) -> FastqSetResponse:
+async def set_allow_additional_fastqs_to_false(fastq_set_id: str = Depends(sanitise_fqs_orcabus_id)) -> FastqSetResponseDict:
     fastq_set_obj = FastqSetData.get(fastq_set_id)
 
     # Check fastq set exists
@@ -736,7 +756,7 @@ async def set_allow_additional_fastqs_to_false(fastq_set_id: str = Depends(sanit
     tags=["fastqset merge"],
     description="Merge multiple fastq sets into a single fastq set"
 )
-async def merge_fastq_sets(fastq_set_ids: MergePatch = Depends()) -> FastqSetResponse:
+async def merge_fastq_sets(fastq_set_ids: MergePatch = Depends()) -> FastqSetResponseDict:
     # Check that the fastq set ids are unique
     fastq_set_ids = fastq_set_ids.fastq_set_ids
     if len(list(set(fastq_set_ids))) != len(fastq_set_ids):
@@ -856,7 +876,7 @@ async def validate_ntsm_internal(fastq_set_id: str = Depends(sanitise_fqs_orcabu
 
     # Get the fastq list rows
     fastq_list_rows = list(map(
-        lambda fastq_obj_iter_: FastqListRowData.get(fastq_obj_iter_.id),
+        lambda fastq_set_id_iter: FastqListRowData.get(fastq_set_id_iter),
         fastq_set_obj.fastq_set_ids
     ))
 
@@ -882,7 +902,7 @@ async def validate_ntsm_internal(fastq_set_id: str = Depends(sanitise_fqs_orcabu
     tags=["fastqset ntsm"],
     description="Validate all fastq list rows in the ntsm match, run all-by-all on the ntsms in the fastq set"
 )
-async def validate_ntsm_internal(fastq_set_id_x: str = Depends(sanitise_fqs_orcabus_id), fastq_set_id_y = Depends(sanitise_fqs_orcabus_id)) -> Dict:
+async def validate_ntsm_internal(fastq_set_id_x: str = Depends(sanitise_fqs_orcabus_id_x), fastq_set_id_y = Depends(sanitise_fqs_orcabus_id_y)) -> Dict:
     # Get the fastq set object
     fastq_set_obj_x = FastqSetData.get(fastq_set_id_x)
     fastq_set_obj_y = FastqSetData.get(fastq_set_id_y)
@@ -896,7 +916,7 @@ async def validate_ntsm_internal(fastq_set_id_x: str = Depends(sanitise_fqs_orca
 
     # Get the fastq list rows
     fastq_list_rows_x = list(map(
-        lambda fastq_obj_iter_: FastqListRowData.get(fastq_obj_iter_.id),
+        lambda fastq_obj_iter_: FastqListRowData.get(fastq_obj_iter_),
         fastq_set_obj_x.fastq_set_ids
     ))
 
@@ -919,7 +939,7 @@ async def validate_ntsm_internal(fastq_set_id_x: str = Depends(sanitise_fqs_orca
 
     # Get the fastq list rows
     fastq_list_rows_y = list(map(
-        lambda fastq_obj_iter_: FastqListRowData.get(fastq_obj_iter_.id),
+        lambda fastq_obj_iter_: FastqListRowData.get(fastq_obj_iter_),
         fastq_set_obj_y.fastq_set_ids
     ))