Feature/validator orchestrator

.github/integration/scripts/charts/values.yaml

@@ -90,8 +90,8 @@ global:
     s3ReadyPath: "/minio/health/ready"
     existingClaim: inbox-pvc
   reencrypt:
-    host: reencrypt
-    port: 50443
+    host: pipeline-sda-svc-reencrypt
+    port: 50051
   sync:
     api:
       password: "apiuser"

rabbitmq/definitions.json

@@ -162,6 +162,19 @@
             "durable": true,
             "auto_delete": false,
             "arguments": {}
+        }, {
+          "name": "validation-job-queue",
+          "vhost": "sda",
+          "durable": true,
+          "auto_delete": false,
+          "arguments": {}
+        },
+        {
+          "name": "validation-job-preparation-queue",
+          "vhost": "sda",
+          "durable": true,
+          "auto_delete": false,
+          "arguments": {}
         }
     ],
     "exchanges": [
@@ -279,6 +292,22 @@
             "arguments": {},
             "destination": "catch_all.dead",
             "routing_key": "#"
+        },
+        {
+          "source": "sda",
+          "vhost": "sda",
+          "destination_type": "queue",
+          "arguments": {},
+          "destination": "validation-job-queue",
+          "routing_key": "validation-job-queue"
+        },
+        {
+          "source": "sda",
+          "vhost": "sda",
+          "destination_type": "queue",
+          "arguments": {},
+          "destination": "validation-job-preparation-queue",
+          "routing_key": "validation-job-preparation-queue"
         }
     ]
 }

sda-validator/apptainer-example/100-200_word_validator.def

@@ -0,0 +1,109 @@
+BootStrap: docker
+From: alpine:3.22
+
+%post
+   apk add --no-cache jq bash
+
+%environment
+   export LC_ALL=C
+
+%runscript
+   #!/bin/bash
+   if [ "$1" != "--describe" ]; then
+
+    allPass=true
+    output='{
+      "files": ['
+
+    anyFiles=false
+
+    for k in $(jq '.files | keys | .[]' /mnt/input/input.json); do
+      anyFiles=true
+      value=$(jq -r ".files[$k]" /mnt/input/input.json);
+      path=$(jq -r '.path' <<< "$value");
+
+      if [ -f $path ]; then
+      wordCount=$(wc -w < $path)
+
+      if (($wordCount >= 100 && $wordCount <= 200)); then
+        output+="
+        {
+          \"path\": \"$path\",
+          \"result\": \"passed\"
+        },"
+      else
+
+        allPass=false
+
+        output+="
+        {
+          \"path\": \"$path\",
+          \"result\": \"failed\",
+          \"messages\": [
+            {
+              \"level\": \"error\",
+              \"time\": \"$(date +"%Y-%m-%dT%H:%M:%S%z")\",
+              \"message\": \"file has $wordCount words, which is not between 100 and 200\"
+            }
+          ]
+        },"
+      fi
+      else
+
+        allPass=false
+
+        output+="
+        {
+          \"path\": \"$path\",
+          \"result\": \"failed\",
+          \"messages\": [
+            {
+              \"level\": \"error\",
+              \"time\": \"$(date +"%Y-%m-%dT%H:%M:%S%z")\",
+              \"message\": \"file does not exists\"
+            }
+          ]
+        },"
+      fi
+    done
+
+    if [ $anyFiles = true ]; then
+    output=${output::-1}
+
+   if [ $allPass = true ]; then
+    output+="
+    ],
+    \"result\": \"passed\"
+   }"
+   else
+    output+="
+      ],
+      \"result\": \"failed\"
+    }"
+   fi
+   else
+      output+="
+      ],
+      \"result\": \"failed\",
+       \"messages\": [
+         {
+           \"level\": \"error\",
+           \"time\": \"$(date +"%Y-%m-%dT%H:%M:%S%z")\",
+           \"message\": \"No files submitted to be validated\"
+         }
+       ]
+     }"
+   fi
+
+    echo $output > /mnt/output/result.json
+
+   else
+     echo '{
+   "validatorId": "validator-word-count-100-200",
+   "name": "File word count validator",
+   "description": "Checks that the word count of a file is between 100 and 200 words",
+   "version": "1.0.0",
+   "mode": "file",
+   "pathSpecification": ["*"]
+   }'
+   fi

sda-validator/apptainer-example/README.md

@@ -0,0 +1,11 @@
+# Apptainer examples
+This contains 2 example [Apptainer](https://apptainer.org/) validators following the [Validator Framework Specification](https://github.com/GenomicDataInfrastructure/validation-framework/blob/main/validator-specification.md)
+
+* [100-200 Word Validator](100-200_word_validator.def) validates that a file has between 100 and 200 words
+* [Always Success Validator](always_success.def) always returns success("passed") on validations, unless no files are provided.  
+
+## Build and compile 
+```bash
+apptainer build 100-200_word_validator.sif 100-200_word_validator.def
+apptainer build always_success.sif always_success.def
+```

sda-validator/apptainer-example/always_success.def

@@ -0,0 +1,90 @@
+BootStrap: docker
+From: alpine:3.22
+
+%post
+   apk add --no-cache jq bash
+
+%environment
+   export LC_ALL=C
+
+%runscript
+   #!/bin/bash
+   if [ "$1" != "--describe" ]; then
+
+    allPass=true
+    output='{
+      "files": ['
+
+    anyFiles=false
+
+    for k in $(jq '.files | keys | .[]' /mnt/input/input.json); do
+      anyFiles=true
+      value=$(jq -r ".files[$k]" /mnt/input/input.json);
+      path=$(jq -r '.path' <<< "$value");
+
+      if [ -f $path ]; then
+
+        output+="
+        {
+          \"path\": \"$path\",
+          \"result\": \"passed\"
+        },"
+      else
+
+        allPass=false
+
+        output+="
+        {
+          \"path\": \"$path\",
+          \"result\": \"failed\",
+          \"messages\": [
+            {
+              \"level\": \"error\",
+              \"time\": \"$(date +"%Y-%m-%dT%H:%M:%S%z")\",
+              \"message\": \"file does not exists\"
+            }
+          ]
+        },"
+      fi
+    done
+
+    if [ $anyFiles = true ]; then
+    output=${output::-1}
+
+   if [ $allPass = true ]; then
+    output+="
+    ],
+    \"result\": \"passed\"
+   }"
+   else
+    output+="
+      ],
+      \"result\": \"failed\"
+    }"
+   fi
+   else
+      output+="
+      ],
+      \"result\": \"failed\",
+       \"messages\": [
+         {
+           \"level\": \"error\",
+           \"time\": \"$(date +"%Y-%m-%dT%H:%M:%S%z")\",
+           \"message\": \"No files submitted to be validated\"
+         }
+       ]
+     }"
+   fi
+
+    echo $output > /mnt/output/result.json
+
+   else
+     echo '{
+   "validatorId": "validator-always-success",
+   "name": "Always success validator",
+   "description": "Validator always succeed",
+   "version": "1.0.0",
+   "mode": "file",
+   "pathSpecification": ["*"]
+   }'
+   fi

sda-validator/orchestrator/.dockerignore

@@ -0,0 +1,5 @@
+swagger_v1.yml
+README.md
+.kube
+Dockerfile
+.dockerignore

sda-validator/orchestrator/.kube/01_service.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: sda-validator-orchestrator
+  labels:
+    app: sda-validator-orchestrator
+spec:
+  ports:
+    - name: api
+      port: 8080
+      targetPort: api
+      protocol: TCP
+  selector:
+    app: sda-validator-orchestrator

sda-validator/orchestrator/.kube/02_rbac_config_map.yaml

@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: sda-validator-orchestrator-rbac
+data:
+  policy.json: |-
+    {
+      "policy": [
+        {
+          "role": "admin",
+          "path": "/admin/*",
+          "action": "(GET)|(POST)"
+        }, {
+          "role": "submission",
+          "path": "/validators",
+          "action": "GET"
+        }, {
+          "role": "submission",
+          "path": "/validate",
+          "action": "POST"
+        }, {
+          "role": "submission",
+          "path": "/result",
+          "action": "GET"
+        }
+      ],
+      "roles": [
+        {
+          "role": "admin",
+          "rolebinding": "submission"
+        },
+        {
+          "role": "[email protected]",
+          "rolebinding": "admin"
+        }, {
+          "role": "[email protected]",
+          "rolebinding": "admin"
+        }
+      ]
+    }

sda-validator/orchestrator/.kube/03_secret.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: sda-validator-orchestrator
+type: Opaque
+stringData:
+  sdaApiToken: ""
+  brokerPassword: ""
+  pgPassword: ""
+