Refactor Base Schema to not require Donor/Specimen/Sample (#878)

* Remove Donor, Specimen, and Sample from Analysis Data Classes in SONG

* Reverting the changes related to latest develop

* Remove Donor, Specimen, and Samples from Base Schema and Allow in Dynamic Schemas

* Remove Donor-Related Entities from StudyController and Search APIs

* Add External Validation Rules for Analysis Properties Based on Dynamic Schemas

* Resolving the build issues and updating the PR with missing files

* Changes related to feedback

* Changes to ValidationService related to externalValiation

* Changes related to the feedback comments

* Migrate file_types to options column in analysis_schema

* External valdiation URL template for study and value tokens

* Retrieve all options data with analysis type

* Remove sneaky throws from buildSchema with JSON parsing failure case

* Fix paginated data lookup by removing stored procedure reference to file_type enum

* Schema registrations sets options values correctly (#879)

- when missing, uses value from previous schema or sets as empty (default) value
- when set as empty array, replaces current value with empty array (removes optional validations)
- sets options in get registration by version query

* Change schema option name to plural `externalValidations`

* Docs grammar improvements and update property to externalValidations

* Remove blank lines

* Options property description improvements

---------

Co-authored-by: Azher2Ali <[email protected]>

Author: joneubank

docs/custom-schemas.md

@@ -0,0 +1,110 @@
+# Custom Analysis Schemas
+
+## Minimal Example
+
+```json
+{
+ "name": "minimalExample",
+ "options":{},
+ "schema":{
+     "type": "object",
+     "required":[
+         "experiment"
+     ],
+     "properties":{
+        "experiment":{}
+     }
+  }
+}
+```
+
+## Options
+
+The `options` property defines extra validations for this analysis schema, such as restrictions on file types and checks on the data with an external service. The `options` property is not required. Similarly, each property in `options` is also optional. If no value is provided for one of the `options` properties, a default configuration will be used for the analysis. If this is an update to an existing analysis type, you can omit any option and its value will be maintained from the previous version.
+
+```json
+{
+	"fileTypes":["bam", "cram"],
+	"externalValidations":[
+		{
+			"url": "http://localhost:8099/",
+			"jsonPath": "experiment.someId"
+		}
+	]
+},
+```
+
+If you want to remove the previous value of an option so that this validation is no longer required, for instance removing the restriction on file types so that any file type could be provided, you should provide an empty list for that option.
+
+In the example below, both `fileTypes` and `externalValidations` properties are set to empty arrays, which means that these validations will not be applied to submitted analysis:
+
+```json
+{
+	"options": {
+		"fileTypes": [],
+		"externalValidations": []
+	}
+}
+```
+
+### File Types
+
+`options.fileTypes` can be provided an array of strings. These represent the file types (file extensions) that are allowed to be uploaded for this type of analysis.
+
+If an empty array is provided, then any file type will be allowed. If an array of file types is provided, then an analysis will be invalid if it contains files of a type not listed.
+
+```json
+{
+	"options": {
+		"fileTypes": ["bam","cram"]
+	}
+}
+```
+
+### External Validation
+
+External validations configure Song to check a value in the analysis against an external service by sending an HTTP GET request to a configurable URL. The service should respond with a 2XX status message to indicate the value "is valid".
+
+As an example, if the project clinical data is being managed in a separate service, we can add an external validation to the donor id field of our custom scheme. This will send the donor id to the external service which can confirm that we have previously registered that donor.
+
+This might look like the following:
+
+```json
+{
+	"url": "http://example.com/{study}/donor/{value}",
+	"jsonPath": "experiment.donorId"
+}
+```
+
+The URL provided is a template, with two variables that will be replaced during validation. Song will replace the token `{value}` with the value read from the analysis at the property as defined in the `jsonPath`. Song will also replace the token `{study}` with the study ID for the Analysis.
+
+Continuing the above example, if the following analysis was submitted:
+
+```json
+{
+  "studyId": "ABC123",
+  "analysisType": {
+    "name": "minimalExample"
+  },
+  "files": [
+    {
+      "dataType": "text",
+      "fileName": "file1.txt",
+      "fileSize": 123,
+      "fileType": "txt",
+      "fileAccess": "open",
+      "fileMd5sum": "595f44fec1e92a71d3e9e77456ba80d1"
+    }
+  ],
+  "experiment": {
+    "donorId": "id01"
+  }
+}
+```
+
+Song would attempt to validate the donorId by sending a validation request to `http://example.com/ABC123/donor/id01`.
+
+The URL parsing allows using either the `{study}` or `{value}` placeholders multiple times (e.g. `http://example.com/{study}-{value}/{value}`), each instance will be interpolated accordingly.
+
+> [!Warning]
+> The URL may cause errors in Song if it contains any tokens matching the `{word}` format other than `{study}` and `{value}`

song-client/src/main/java/bio/overture/song/client/command/SearchCommand.java

@@ -54,31 +54,13 @@ public class SearchCommand extends Command {
   /** Long Switch Constants */
   private static final String FILE_ID_SWITCH = "--file-id";
 
-  private static final String SAMPLE_ID_SWITCH = "--sample-id";
-  private static final String SPECIMEN_ID_SWITCH = "--specimen-id";
-  private static final String DONOR_ID_SWITCH = "--donor-id";
   private static final String ANALYSIS_ID_SWITCH = "--analysis-id";
 
   @Parameter(
       names = {F_SWITCH, FILE_ID_SWITCH},
       required = false)
   private String fileId;
 
-  @Parameter(
-      names = {SA_SWITCH, SAMPLE_ID_SWITCH},
-      required = false)
-  private String sampleId;
-
-  @Parameter(
-      names = {SP_SWITCH, SPECIMEN_ID_SWITCH},
-      required = false)
-  private String specimenId;
-
-  @Parameter(
-      names = {D_SWITCH, DONOR_ID_SWITCH},
-      required = false)
-  private String donorId;
-
   @Parameter(
       names = {A_SWITCH, ANALYSIS_ID_SWITCH},
       required = false)
@@ -92,8 +74,7 @@ public void run() throws IOException {
     val status = checkRules();
     if (!status.hasErrors()) {
       if (isIdSearchMode()) {
-        status.outputPrettyJson(
-            songApi.idSearch(config.getStudyId(), sampleId, specimenId, donorId, fileId));
+        status.outputPrettyJson(songApi.idSearch(config.getStudyId(), fileId));
       } else if (isAnalysisSearchMode()) {
         status.outputPrettyJson(songApi.getAnalysis(config.getStudyId(), analysisId));
       } else {
@@ -108,18 +89,15 @@ private boolean isAnalysisSearchMode() {
   }
 
   private boolean isIdSearchMode() {
-    return nonNull(fileId) || nonNull(sampleId) || nonNull(specimenId) || nonNull(donorId);
+    return nonNull(fileId);
   }
 
   private Status checkRules() {
     val fileTerm = createParamTerm(F_SWITCH, FILE_ID_SWITCH, fileId, Objects::nonNull);
-    val sampleTerm = createParamTerm(SA_SWITCH, SAMPLE_ID_SWITCH, sampleId, Objects::nonNull);
-    val specimenTerm = createParamTerm(SP_SWITCH, SPECIMEN_ID_SWITCH, specimenId, Objects::nonNull);
-    val donorTerm = createParamTerm(D_SWITCH, DONOR_ID_SWITCH, donorId, Objects::nonNull);
     val analysisIdTerm =
         createParamTerm(A_SWITCH, ANALYSIS_ID_SWITCH, analysisId, Objects::nonNull);
 
-    val idSearchMode = createModeRule(ID_MODE, fileTerm, sampleTerm, specimenTerm, donorTerm);
+    val idSearchMode = createModeRule(ID_MODE, fileTerm);
     val analysisSearchMode = createModeRule(ANALYSIS_MODE, analysisIdTerm);
     val ruleProcessor = createRuleProcessor(idSearchMode, analysisSearchMode);
     return ruleProcessor.check();

song-client/src/test/java/bio/overture/song/client/HappyPathClientMainTest.java

@@ -39,13 +39,10 @@
 import bio.overture.song.core.model.Analysis;
 import bio.overture.song.core.model.AnalysisType;
 import bio.overture.song.core.model.AnalysisTypeId;
-import bio.overture.song.core.model.Donor;
 import bio.overture.song.core.model.FileDTO;
 import bio.overture.song.core.model.FileUpdateRequest;
 import bio.overture.song.core.model.FileUpdateResponse;
 import bio.overture.song.core.model.PageDTO;
-import bio.overture.song.core.model.Sample;
-import bio.overture.song.core.model.Specimen;
 import bio.overture.song.core.model.SubmitResponse;
 import bio.overture.song.core.utils.RandomGenerator;
 import bio.overture.song.sdk.ManifestClient;
@@ -280,9 +277,6 @@ public void testAnalysisSearch() {
   @Test
   public void testIdSearch() {
     val expectedFile = new FileDTO().setObjectId("FI1");
-    val expectedSample = Sample.builder().sampleId("SA1").build();
-    val expectedSpecimen = Specimen.builder().specimenId("SP1").build();
-    val expectedDonor = Donor.builder().donorId("DO1").build();
     val expectedAnalyses =
         List.of(
             Analysis.builder()
@@ -292,27 +286,7 @@ public void testIdSearch() {
                 .studyId(DUMMY_STUDY_ID)
                 .build());
 
-    when(songApi.idSearch(DUMMY_STUDY_ID, expectedSample.getSampleId(), null, null, null))
-        .thenReturn(expectedAnalyses);
-    when(songApi.idSearch(DUMMY_STUDY_ID, null, expectedSpecimen.getSpecimenId(), null, null))
-        .thenReturn(expectedAnalyses);
-    when(songApi.idSearch(DUMMY_STUDY_ID, null, null, expectedDonor.getDonorId(), null))
-        .thenReturn(expectedAnalyses);
-    when(songApi.idSearch(DUMMY_STUDY_ID, null, null, null, expectedFile.getObjectId()))
-        .thenReturn(expectedAnalyses);
-    assertOutputJson(objectToTree(expectedAnalyses), "search", "-d", expectedDonor.getDonorId());
-    assertOutputJson(
-        objectToTree(expectedAnalyses), "search", "--donor-id", expectedDonor.getDonorId());
-    assertOutputJson(
-        objectToTree(expectedAnalyses), "search", "-sp", expectedSpecimen.getSpecimenId());
-    assertOutputJson(
-        objectToTree(expectedAnalyses),
-        "search",
-        "--specimen-id",
-        expectedSpecimen.getSpecimenId());
-    assertOutputJson(objectToTree(expectedAnalyses), "search", "-sa", expectedSample.getSampleId());
-    assertOutputJson(
-        objectToTree(expectedAnalyses), "search", "--sample-id", expectedSample.getSampleId());
+    when(songApi.idSearch(DUMMY_STUDY_ID, expectedFile.getObjectId())).thenReturn(expectedAnalyses);
     assertOutputJson(objectToTree(expectedAnalyses), "search", "-f", expectedFile.getObjectId());
     assertOutputJson(
         objectToTree(expectedAnalyses), "search", "--file-id", expectedFile.getObjectId());

song-core/src/main/java/bio/overture/song/core/exceptions/ServerErrors.java

@@ -34,9 +34,6 @@
 
 public enum ServerErrors implements ServerError {
   STUDY_ID_DOES_NOT_EXIST(NOT_FOUND),
-  DONOR_DOES_NOT_EXIST(NOT_FOUND),
-  SPECIMEN_DOES_NOT_EXIST(NOT_FOUND),
-  SAMPLE_DOES_NOT_EXIST(NOT_FOUND),
   UPLOAD_REPOSITORY_CREATE_RECORD(UNPROCESSABLE_ENTITY),
   ANALYSIS_REPOSITORY_CREATE_RECORD(UNPROCESSABLE_ENTITY),
   SEQUENCING_READ_REPOSITORY_CREATE_RECORD(UNPROCESSABLE_ENTITY),
@@ -45,18 +42,9 @@ public enum ServerErrors implements ServerError {
   INFO_REPOSITORY_UPDATE_RECORD(UNPROCESSABLE_ENTITY),
   INFO_REPOSITORY_DELETE_RECORD(UNPROCESSABLE_ENTITY),
   FILE_REPOSITORY_UPDATE_RECORD(UNPROCESSABLE_ENTITY),
-  DONOR_REPOSITORY_UPDATE_RECORD(UNPROCESSABLE_ENTITY),
-  SAMPLE_REPOSITORY_UPDATE_RECORD(UNPROCESSABLE_ENTITY),
-  SPECIMEN_REPOSITORY_UPDATE_RECORD(UNPROCESSABLE_ENTITY),
   STUDY_REPOSITORY_CREATE_RECORD(UNPROCESSABLE_ENTITY),
   FILE_REPOSITORY_DELETE_RECORD(UNPROCESSABLE_ENTITY),
-  DONOR_REPOSITORY_DELETE_RECORD(UNPROCESSABLE_ENTITY),
-  SPECIMEN_REPOSITORY_DELETE_RECORD(UNPROCESSABLE_ENTITY),
-  SAMPLE_REPOSITORY_DELETE_RECORD(UNPROCESSABLE_ENTITY),
   GENERATOR_CLOCK_MOVED_BACKWARDS(INTERNAL_SERVER_ERROR),
-  DONOR_ID_IS_CORRUPTED(INTERNAL_SERVER_ERROR),
-  SAMPLE_ID_IS_CORRUPTED(INTERNAL_SERVER_ERROR),
-  SPECIMEN_ID_IS_CORRUPTED(INTERNAL_SERVER_ERROR),
   PAYLOAD_PARSING(UNPROCESSABLE_ENTITY),
   UPLOAD_ID_NOT_FOUND(NOT_FOUND),
   FILE_NOT_FOUND(NOT_FOUND),
@@ -68,7 +56,6 @@ public enum ServerErrors implements ServerError {
   UPLOAD_ID_NOT_VALIDATED(CONFLICT),
   ANALYSIS_ID_NOT_CREATED(INTERNAL_SERVER_ERROR),
   ANALYSIS_MISSING_FILES(INTERNAL_SERVER_ERROR),
-  ANALYSIS_MISSING_SAMPLES(INTERNAL_SERVER_ERROR),
   ANALYSIS_ID_NOT_FOUND(NOT_FOUND),
   SEQUENCING_READ_NOT_FOUND(NOT_FOUND),
   VARIANT_CALL_NOT_FOUND(NOT_FOUND),
@@ -80,23 +67,12 @@ public enum ServerErrors implements ServerError {
   GATEWAY_SERVICE_NOT_FOUND(BAD_GATEWAY),
   SERVICE_UNAVAILABLE(HttpStatus.SERVICE_UNAVAILABLE),
   NOT_IMPLEMENTED_YET(NOT_IMPLEMENTED),
-  SAMPLE_REPOSITORY_CREATE_RECORD(INTERNAL_SERVER_ERROR),
   FILE_REPOSITORY_CREATE_RECORD(INTERNAL_SERVER_ERROR),
-  SPECIMEN_RECORD_FAILED(INTERNAL_SERVER_ERROR),
-  DONOR_RECORD_FAILED(INTERNAL_SERVER_ERROR),
   ANALYSIS_STATE_UPDATE_FAILED(INTERNAL_SERVER_ERROR),
   SEARCH_TERM_SYNTAX(BAD_REQUEST),
   ANALYSIS_TYPE_INCORRECT_VERSION(CONFLICT),
   ANALYSIS_ID_COLLISION(CONFLICT),
   INFO_ALREADY_EXISTS(CONFLICT),
-  SAMPLE_ALREADY_EXISTS(CONFLICT),
-  DONOR_ALREADY_EXISTS(CONFLICT),
-  SPECIMEN_ALREADY_EXISTS(CONFLICT),
-  SAMPLE_TO_SPECIMEN_ID_MISMATCH(CONFLICT),
-  MISMATCHING_SAMPLE_DATA(CONFLICT),
-  MISMATCHING_SPECIMEN_DATA(CONFLICT),
-  MISMATCHING_DONOR_DATA(CONFLICT),
-  SPECIMEN_TO_DONOR_ID_MISMATCH(CONFLICT),
   VARIANT_CALL_CORRUPTED_DUPLICATE(INTERNAL_SERVER_ERROR),
   SEQUENCING_READ_CORRUPTED_DUPLICATE(INTERNAL_SERVER_ERROR),
   STUDY_ALREADY_EXISTS(CONFLICT),

song-core/src/main/java/bio/overture/song/core/model/Analysis.java

@@ -23,7 +23,6 @@ public class Analysis extends DynamicData {
   private String studyId;
   private AnalysisStates analysisState;
   private AnalysisTypeId analysisType;
-  private List<CompositeSample> samples;
   private List<FileDTO> files;
 
   private LocalDateTime createdAt;

song-core/src/main/java/bio/overture/song/core/model/AnalysisType.java

@@ -3,6 +3,8 @@
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.databind.JsonNode;
 import java.time.LocalDateTime;
+import java.util.ArrayList;
+import java.util.List;
 import javax.validation.constraints.NotNull;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
@@ -22,4 +24,24 @@ public class AnalysisType {
 
   @JsonInclude(JsonInclude.Include.NON_NULL)
   private JsonNode schema;
+
+  public List<String> getFileTypes() {
+    if (this.options == null) {
+      return new ArrayList<String>();
+    }
+    if (this.options.getFileTypes() == null) {
+      return new ArrayList<String>();
+    }
+    return this.options.getFileTypes();
+  }
+
+  public List<ExternalValidation> getExternalValidations() {
+    if (this.options == null) {
+      return new ArrayList<ExternalValidation>();
+    }
+    if (this.options.getExternalValidations() == null) {
+      return new ArrayList<ExternalValidation>();
+    }
+    return this.options.getExternalValidations();
+  }
 }

song-core/src/main/java/bio/overture/song/core/model/AnalysisTypeOptions.java

@@ -1,12 +1,16 @@
 package bio.overture.song.core.model;
 
 import java.util.List;
-import lombok.*;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
 
 @Data
 @Builder
 @NoArgsConstructor
 @AllArgsConstructor
 public class AnalysisTypeOptions {
   private List<String> fileTypes;
+  private List<ExternalValidation> externalValidations;
 }