From 9ec6d2e8bb8ea9d2dc0cefe849deee7f2f8f4c21 Mon Sep 17 00:00:00 2001 From: aniewielska Date: Mon, 26 Feb 2018 15:02:01 +0000 Subject: [PATCH 1/9] Taskmaster input for examples --- .../error_stops_execution_taskmaster.json | 113 ++++++++++++++++++ examples/error/executor_taskmaster.json | 45 +++++++ examples/error/input_taskmaster.json | 84 +++++++++++++ examples/success/env_taskmaster.json | 100 ++++++++++++++++ examples/success/hello_taskmaster.json | 46 +++++++ examples/success/stderr_taskmaster.json | 81 +++++++++++++ examples/success/stdin_taskmaster.json | 45 +++++++ examples/success/stdout_taskmaster.json | 81 +++++++++++++ examples/success/tags_taskmaster.json | 47 ++++++++ examples/success/workdir_taskmaster.json | 44 +++++++ 10 files changed, 686 insertions(+) create mode 100644 examples/error/error_stops_execution_taskmaster.json create mode 100644 examples/error/executor_taskmaster.json create mode 100644 examples/error/input_taskmaster.json create mode 100644 examples/success/env_taskmaster.json create mode 100644 examples/success/hello_taskmaster.json create mode 100644 examples/success/stderr_taskmaster.json create mode 100644 examples/success/stdin_taskmaster.json create mode 100644 examples/success/stdout_taskmaster.json create mode 100644 examples/success/tags_taskmaster.json create mode 100644 examples/success/workdir_taskmaster.json diff --git a/examples/error/error_stops_execution_taskmaster.json b/examples/error/error_stops_execution_taskmaster.json new file mode 100644 index 00000000..55f01c5d --- /dev/null +++ b/examples/error/error_stops_execution_taskmaster.json @@ -0,0 +1,113 @@ +{ + "outputs": [], + "inputs": [], + "volumes": [], + "executors": [ + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-ec104c85", + "executor-no": "0" + }, + "name": "task-ec104c85-ex-00" + }, + "spec": { + "template": { + "metadata": { + "name": "task-ec104c85-ex-00" + }, + "spec": { + "containers": [ + { + "command": [ + "echo", + "You will see this in the logs (stdout)." + ], + "image": "alpine", + "name": "task-ec104c85-ex-00", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + }, + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-ec104c85", + "executor-no": "1" + }, + "name": "task-ec104c85-ex-01" + }, + "spec": { + "template": { + "metadata": { + "name": "task-ec104c85-ex-01" + }, + "spec": { + "containers": [ + { + "command": [ + "sh", + "-c", + "exit 1" + ], + "image": "alpine", + "name": "task-ec104c85-ex-01", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + }, + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-ec104c85", + "executor-no": "2" + }, + "name": "task-ec104c85-ex-02" + }, + "spec": { + "template": { + "metadata": { + "name": "task-ec104c85-ex-02" + }, + "spec": { + "containers": [ + { + "command": [ + "echo", + "This shouldn't appear in the logs (stdout)." + ], + "image": "alpine", + "name": "task-ec104c85-ex-02", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + } + ], + "resources": { + "disk_gb": 0.1 + } +} \ No newline at end of file diff --git a/examples/error/executor_taskmaster.json b/examples/error/executor_taskmaster.json new file mode 100644 index 00000000..8e5f4bf5 --- /dev/null +++ b/examples/error/executor_taskmaster.json @@ -0,0 +1,45 @@ +{ + "outputs": [], + "inputs": [], + "volumes": [], + "executors": [ + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-23bf5e64", + "executor-no": "0" + }, + "name": "task-23bf5e64-ex-00" + }, + "spec": { + "template": { + "metadata": { + "name": "task-23bf5e64-ex-00" + }, + "spec": { + "containers": [ + { + "command": [ + "sh", + "-c", + "exit 1" + ], + "image": "alpine", + "name": "task-23bf5e64-ex-00", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + } + ], + "resources": { + "disk_gb": 0.1 + } +} \ No newline at end of file diff --git a/examples/error/input_taskmaster.json b/examples/error/input_taskmaster.json new file mode 100644 index 00000000..6df439cf --- /dev/null +++ b/examples/error/input_taskmaster.json @@ -0,0 +1,84 @@ +{ + "outputs": [], + "inputs": [ + { + "url": "http://nonexistent.ebi.ac.uk", + "path": "/somewhere/file", + "type": "FILE" + } + ], + "volumes": [], + "executors": [ + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-7ff304bd", + "executor-no": "0" + }, + "name": "task-7ff304bd-ex-00" + }, + "spec": { + "template": { + "metadata": { + "name": "task-7ff304bd-ex-00" + }, + "spec": { + "containers": [ + { + "command": [ + "echo", + "This shouldn't appear in the logs." + ], + "image": "alpine", + "name": "task-7ff304bd-ex-00", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + }, + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-7ff304bd", + "executor-no": "1" + }, + "name": "task-7ff304bd-ex-01" + }, + "spec": { + "template": { + "metadata": { + "name": "task-7ff304bd-ex-01" + }, + "spec": { + "containers": [ + { + "command": [ + "cat", + "/somewhere/file" + ], + "image": "alpine", + "name": "task-7ff304bd-ex-01", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + } + ], + "resources": { + "disk_gb": 0.1 + } +} \ No newline at end of file diff --git a/examples/success/env_taskmaster.json b/examples/success/env_taskmaster.json new file mode 100644 index 00000000..877e5272 --- /dev/null +++ b/examples/success/env_taskmaster.json @@ -0,0 +1,100 @@ +{ + "outputs": [], + "inputs": [], + "volumes": [], + "executors": [ + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-9d70f812", + "executor-no": "0" + }, + "name": "task-9d70f812-ex-00" + }, + "spec": { + "template": { + "metadata": { + "name": "task-9d70f812-ex-00" + }, + "spec": { + "containers": [ + { + "command": [ + "echo", + "$SECRET_PROJECT_NAME", + "$PROJECT_STATUS" + ], + "env": [ + { + "name": "SECRET_PROJECT_NAME", + "value": "TESK" + }, + { + "name": "PROJECT_STATUS", + "value": "rocks!" + } + ], + "image": "alpine", + "name": "task-9d70f812-ex-00", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + }, + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-9d70f812", + "executor-no": "1" + }, + "name": "task-9d70f812-ex-01" + }, + "spec": { + "template": { + "metadata": { + "name": "task-9d70f812-ex-01" + }, + "spec": { + "containers": [ + { + "command": [ + "sh", + "-c", + "echo $SECRET_PROJECT_NAME $PROJECT_STATUS" + ], + "env": [ + { + "name": "SECRET_PROJECT_NAME", + "value": "TESK" + }, + { + "name": "PROJECT_STATUS", + "value": "rocks!" + } + ], + "image": "alpine", + "name": "task-9d70f812-ex-01", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + } + ], + "resources": { + "disk_gb": 0.1 + } +} \ No newline at end of file diff --git a/examples/success/hello_taskmaster.json b/examples/success/hello_taskmaster.json new file mode 100644 index 00000000..4ee5b00d --- /dev/null +++ b/examples/success/hello_taskmaster.json @@ -0,0 +1,46 @@ +{ + "outputs": [], + "inputs": [], + "volumes": [], + "executors": [ + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": { + "tes-task-name": "Hello World" + }, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-7d5c53f4", + "executor-no": "0" + }, + "name": "task-7d5c53f4-ex-00" + }, + "spec": { + "template": { + "metadata": { + "name": "task-7d5c53f4-ex-00" + }, + "spec": { + "containers": [ + { + "command": [ + "echo", + "TESK says: Hello World" + ], + "image": "alpine", + "name": "task-7d5c53f4-ex-00", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + } + ], + "resources": { + "disk_gb": 0.1 + } +} \ No newline at end of file diff --git a/examples/success/stderr_taskmaster.json b/examples/success/stderr_taskmaster.json new file mode 100644 index 00000000..d46433fc --- /dev/null +++ b/examples/success/stderr_taskmaster.json @@ -0,0 +1,81 @@ +{ + "outputs": [], + "inputs": [], + "volumes": [ + "/outputs" + ], + "executors": [ + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-f1150292", + "executor-no": "0" + }, + "name": "task-f1150292-ex-00" + }, + "spec": { + "template": { + "metadata": { + "name": "task-f1150292-ex-00" + }, + "spec": { + "containers": [ + { + "command": [ + "/bin/sh", + "-c", + "sh -c 'grep bash /etc/s* || exit 0' 2> /outputs/stderr" + ], + "image": "ubuntu", + "name": "task-f1150292-ex-00", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + }, + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-f1150292", + "executor-no": "1" + }, + "name": "task-f1150292-ex-01" + }, + "spec": { + "template": { + "metadata": { + "name": "task-f1150292-ex-01" + }, + "spec": { + "containers": [ + { + "command": [ + "cat", + "/outputs/stderr" + ], + "image": "alpine", + "name": "task-f1150292-ex-01", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + } + ], + "resources": { + "disk_gb": 0.1 + } +} \ No newline at end of file diff --git a/examples/success/stdin_taskmaster.json b/examples/success/stdin_taskmaster.json new file mode 100644 index 00000000..d479faeb --- /dev/null +++ b/examples/success/stdin_taskmaster.json @@ -0,0 +1,45 @@ +{ + "outputs": [], + "inputs": [], + "volumes": [], + "executors": [ + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-54d04d9d", + "executor-no": "0" + }, + "name": "task-54d04d9d-ex-00" + }, + "spec": { + "template": { + "metadata": { + "name": "task-54d04d9d-ex-00" + }, + "spec": { + "containers": [ + { + "command": [ + "/bin/sh", + "-c", + "cat < /etc/fstab" + ], + "image": "alpine", + "name": "task-54d04d9d-ex-00", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + } + ], + "resources": { + "disk_gb": 0.1 + } +} \ No newline at end of file diff --git a/examples/success/stdout_taskmaster.json b/examples/success/stdout_taskmaster.json new file mode 100644 index 00000000..a7568a7d --- /dev/null +++ b/examples/success/stdout_taskmaster.json @@ -0,0 +1,81 @@ +{ + "outputs": [], + "inputs": [], + "volumes": [ + "/outputs" + ], + "executors": [ + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-04512e8f", + "executor-no": "0" + }, + "name": "task-04512e8f-ex-00" + }, + "spec": { + "template": { + "metadata": { + "name": "task-04512e8f-ex-00" + }, + "spec": { + "containers": [ + { + "command": [ + "/bin/sh", + "-c", + "echo 'This will appear in stdout, but of the 2. executor.' > /outputs/stdout" + ], + "image": "ubuntu", + "name": "task-04512e8f-ex-00", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + }, + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-04512e8f", + "executor-no": "1" + }, + "name": "task-04512e8f-ex-01" + }, + "spec": { + "template": { + "metadata": { + "name": "task-04512e8f-ex-01" + }, + "spec": { + "containers": [ + { + "command": [ + "cat", + "/outputs/stdout" + ], + "image": "alpine", + "name": "task-04512e8f-ex-01", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + } + ], + "resources": { + "disk_gb": 0.1 + } +} \ No newline at end of file diff --git a/examples/success/tags_taskmaster.json b/examples/success/tags_taskmaster.json new file mode 100644 index 00000000..93df8e20 --- /dev/null +++ b/examples/success/tags_taskmaster.json @@ -0,0 +1,47 @@ +{ + "outputs": [], + "inputs": [], + "volumes": [], + "executors": [ + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": { + "tes-task-name": "I do nothing" + }, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-fc67859b", + "executor-no": "0" + }, + "name": "task-fc67859b-ex-00" + }, + "spec": { + "template": { + "metadata": { + "name": "task-fc67859b-ex-00" + }, + "spec": { + "containers": [ + { + "command": [ + "sh", + "-c", + "exit 0" + ], + "image": "alpine", + "name": "task-fc67859b-ex-00", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + } + ], + "resources": { + "disk_gb": 0.1 + } +} \ No newline at end of file diff --git a/examples/success/workdir_taskmaster.json b/examples/success/workdir_taskmaster.json new file mode 100644 index 00000000..346352c7 --- /dev/null +++ b/examples/success/workdir_taskmaster.json @@ -0,0 +1,44 @@ +{ + "outputs": [], + "inputs": [], + "volumes": [], + "executors": [ + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-49299ae5", + "executor-no": "0" + }, + "name": "task-49299ae5-ex-00" + }, + "spec": { + "template": { + "metadata": { + "name": "task-49299ae5-ex-00" + }, + "spec": { + "containers": [ + { + "command": [ + "ls" + ], + "image": "alpine", + "name": "task-49299ae5-ex-00", + "resources": {}, + "workingDir": "/etc" + } + ], + "restartPolicy": "Never" + } + } + } + } + ], + "resources": { + "disk_gb": 0.1 + } +} \ No newline at end of file From f91102eaa485f5fdadbbfedea5bf34d5f8c3e694 Mon Sep 17 00:00:00 2001 From: Pau Ruiz i Safont Date: Tue, 27 Feb 2018 10:40:06 +0000 Subject: [PATCH 2/9] maintenance: change hashbang and imports The hashbang now explicitely states that python2 is needed, as well as using env to run the first python2 binary in $PATH The imports were reordered as recommended by pylint --- scripts/exportsecrets.py | 4 ++-- scripts/filer.py | 4 ++-- scripts/genjob.py | 2 +- scripts/state.py | 8 ++++---- scripts/taskmaster.py | 14 ++++++-------- 5 files changed, 15 insertions(+), 17 deletions(-) diff --git a/scripts/exportsecrets.py b/scripts/exportsecrets.py index dcb93fe0..5a7de89c 100755 --- a/scripts/exportsecrets.py +++ b/scripts/exportsecrets.py @@ -1,7 +1,7 @@ -#!/usr/bin/python +#!/usr/bin/env python2 -import yaml import os +import yaml config = yaml.safe_load(open(os.path.join(os.environ["HOME"], '.kube/config'))) diff --git a/scripts/filer.py b/scripts/filer.py index bfeba853..43e129d6 100755 --- a/scripts/filer.py +++ b/scripts/filer.py @@ -1,15 +1,15 @@ -#!/usr/bin/python +#!/usr/bin/env python2 from __future__ import print_function from ftplib import FTP import ftplib import argparse -import requests import sys import json import re import os import distutils.dir_util +import requests debug = True diff --git a/scripts/genjob.py b/scripts/genjob.py index 5b4228ba..95d75a0c 100755 --- a/scripts/genjob.py +++ b/scripts/genjob.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python2 import argparse import json diff --git a/scripts/state.py b/scripts/state.py index 128cab41..9006e87d 100755 --- a/scripts/state.py +++ b/scripts/state.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python2 from __future__ import print_function @@ -17,11 +17,11 @@ def exec_state(exe_f, namespace, state): job = bv1.read_namespaced_job(exe, namespace=namespace) state['logs'][exe_i]['start_time'] = job.metadata.creation_timestamp state['logs'][exe_i]['end_time'] = job.status.completion_time - + job_label_s = 'controller-uid='+job.spec.selector.match_labels['controller-uid'] pods = cv1.list_namespaced_pod(label_selector=label_s, namespace=namespace) - try: + try: state['logs'][exe_i]['stdout'] = read_namespaced_pod_log(pods[0].metadata.name, namespace=namespace) except IndexError: print("No pod matching job "+job.metadata.name+" could be found", file=sys.stderr) @@ -46,7 +46,7 @@ def control_state(taskm_f, namespace, state): job_label_s = 'controller-uid='+job.spec.selector.match_labels['controller-uid'] pods = cv1.list_namespaced_pod(label_selector=label_s, namespace=namespace) - try: + try: state['system_logs'] = read_namespaced_pod_log(pods[0].metadata.name, namespace=namespace) except IndexError: print("No pod matching job "+job.metadata.name+" could be found", file=sys.stderr) diff --git a/scripts/taskmaster.py b/scripts/taskmaster.py index aa9769ef..217fc746 100755 --- a/scripts/taskmaster.py +++ b/scripts/taskmaster.py @@ -1,15 +1,13 @@ -#!/usr/bin/python +#!/usr/bin/env python2 from __future__ import print_function import argparse import json import os -import binascii import re import time import sys from kubernetes import client, config -from datetime import datetime created_jobs = [] debug = False @@ -62,10 +60,10 @@ def run_executors(specs, namespace, volume_mounts=None, pvc_name=None): if volume_mounts is not None: spec['containers'][0]['volumeMounts'] = volume_mounts if pvc_name is not None: - spec['volumes'] = [{ 'name': task_volume_basename, 'persistentVolumeClaim': { 'readonly': False, 'claimName': pvc_name }}] + spec['volumes'] = [{'name': task_volume_basename, 'persistentVolumeClaim': { 'readonly': False, 'claimName': pvc_name }}] job = v1.create_namespaced_job(body=executor, namespace=namespace) - + global created_jobs created_jobs.append(jobname) @@ -208,7 +206,7 @@ def cleanup_pvc(data, namespace, volume_mounts, pvc_name, filer_version): container = posttask['spec']['template']['spec']['containers'][0] container['env'].append({ "name": "JSON_INPUT", "value": json.dumps(data) }) container['args'] += ["outputs", "$(JSON_INPUT)"] - + # insert volume mounts and pvc into posttask job template container['volumeMounts'] = volume_mounts posttask['spec']['template']['spec']['volumes'] = [ { "name": task_volume_basename, "persistentVolumeClaim": { "claimName": pvc_name} } ] @@ -286,9 +284,9 @@ def main(argv): def clean_on_interrupt(): print('Caught interrupt signal, deleting jobs and pvc', file=sys.stderr) bv1 = client.BatchV1Api() - + for job in created_jobs: - bv1.delete_namespaced_job(job, args.namespace, client.V1DeleteOptions()) + bv1.delete_namespaced_job(job, args.namespace, client.V1DeleteOptions()) if created_pvc: cv1 = client.CoreV1Api() From 4b82f385b62d9c39af983811e04c47554a238ab8 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Wed, 28 Feb 2018 15:08:47 +0000 Subject: [PATCH 3/9] Clarify a step for minikube users --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8268c64f..13fa25b9 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ For organisational reasons, this project is split into 2 repositories: One conta ## How to install - Clone the repo to your kube-master and cd into the folder - - Edit the following line in `specs/ingress/nginx-ingress-lb.yaml`: + - Find out what is the external IP for the cluster. E.g. with the command `minikube ip` + - Edit the following line in `specs/ingress/nginx-ingress-lb.yaml` with that IP: ```yaml spec: @@ -20,7 +21,7 @@ For organisational reasons, this project is split into 2 repositories: One conta ``` - - Then create the services necessary to run the API: + - Create the services necessary to run the API: ``` kubectl create -f specs/ingress/ From b9b7285ef29a451acaf3a92f613de105cca47a91 Mon Sep 17 00:00:00 2001 From: aniewielska Date: Wed, 28 Feb 2018 14:43:34 +0000 Subject: [PATCH 4/9] Input examples --- .../success/input_directory_ftp01_tree.json | 53 +++++++++++++++++++ .../success/input_directory_ftp02_sub.json | 35 ++++++++++++ .../input_directory_ftp03_tree_shadowing.json | 35 ++++++++++++ .../input_directory_ftp04_tree_shadowing.json | 35 ++++++++++++ .../success/input_directory_ftp05_merge.json | 35 ++++++++++++ .../success/input_file_directory_merge.json | 25 +++++++++ examples/success/input_file_http.json | 19 +++++++ .../input_file_http_nested_mounts.json | 25 +++++++++ 8 files changed, 262 insertions(+) create mode 100644 examples/success/input_directory_ftp01_tree.json create mode 100644 examples/success/input_directory_ftp02_sub.json create mode 100644 examples/success/input_directory_ftp03_tree_shadowing.json create mode 100644 examples/success/input_directory_ftp04_tree_shadowing.json create mode 100644 examples/success/input_directory_ftp05_merge.json create mode 100644 examples/success/input_file_directory_merge.json create mode 100644 examples/success/input_file_http.json create mode 100644 examples/success/input_file_http_nested_mounts.json diff --git a/examples/success/input_directory_ftp01_tree.json b/examples/success/input_directory_ftp01_tree.json new file mode 100644 index 00000000..4b23f038 --- /dev/null +++ b/examples/success/input_directory_ftp01_tree.json @@ -0,0 +1,53 @@ +{ + "description": "Demonstrates retrieving 2 independent FTP directory trees (with subfolders). If the FTP is stable, 1. executor should output dos (!!) and mac subfolders with number of files in them (mac 7, dos 3). We will also list the files in both dirs, to have sth to compare with for next examples", + "inputs": [ + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/tools", + "path": "/tes/old-tools", + "type": "DIRECTORY" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/das", + "path": "/elsewhere", + "type": "DIRECTORY" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\; | sort -nr" + ], + "workdir": "/tes/old-tools" + }, + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\;" + ], + "workdir": "/elsewhere" + }, + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type f" + ], + "workdir": "/tes/old-tools" + }, + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type f" + ], + "workdir": "/elsewhere" + } + ] +} diff --git a/examples/success/input_directory_ftp02_sub.json b/examples/success/input_directory_ftp02_sub.json new file mode 100644 index 00000000..326b0aa7 --- /dev/null +++ b/examples/success/input_directory_ftp02_sub.json @@ -0,0 +1,35 @@ +{ + "description": "Uses FTP directories from previous example (remember dos and mac?). This time 2. directory is mounted below mountpoint for the 1. directory (nested). If the FTP is stable, should output dos (3), mac (7) and windows (3) subfolders", + "inputs": [ + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/tools", + "path": "/tes/old-tools", + "type": "DIRECTORY" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/das", + "path": "/tes/old-tools/windows", + "type": "DIRECTORY" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\; | sort -nr" + ], + "workdir": "/tes/old-tools" + }, + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type f" + ], + "workdir": "/tes/old-tools" + } + ] +} diff --git a/examples/success/input_directory_ftp03_tree_shadowing.json b/examples/success/input_directory_ftp03_tree_shadowing.json new file mode 100644 index 00000000..a69c057b --- /dev/null +++ b/examples/success/input_directory_ftp03_tree_shadowing.json @@ -0,0 +1,35 @@ +{ + "description": "Uses FTP directories from previous examples (dos and mac). This time, we mount second source directory in place of existing directory (dos, that is). The content coming from 'ftp://ftp.ebi.ac.uk/pub/software/tools/dos' and 'ftp://ftp.ebi.ac.uk/pub/software/das' will be merged. Expect 6 files in dos directory.", + "inputs": [ + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/tools", + "path": "/tes/old-tools", + "type": "DIRECTORY" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/das", + "path": "/tes/old-tools/dos", + "type": "DIRECTORY" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\; | sort -nr" + ], + "workdir": "/tes/old-tools" + }, + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type f" + ], + "workdir": "/tes/old-tools" + } + ] +} diff --git a/examples/success/input_directory_ftp04_tree_shadowing.json b/examples/success/input_directory_ftp04_tree_shadowing.json new file mode 100644 index 00000000..13812dc0 --- /dev/null +++ b/examples/success/input_directory_ftp04_tree_shadowing.json @@ -0,0 +1,35 @@ +{ + "description": "The same as previous (03), but proves order does not matter.", + "inputs": [ + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/das", + "path": "/tes/old-tools/dos", + "type": "DIRECTORY" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/tools", + "path": "/tes/old-tools", + "type": "DIRECTORY" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\; | sort -nr" + ], + "workdir": "/tes/old-tools" + }, + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type f" + ], + "workdir": "/tes/old-tools" + } + ] +} diff --git a/examples/success/input_directory_ftp05_merge.json b/examples/success/input_directory_ftp05_merge.json new file mode 100644 index 00000000..06fa6109 --- /dev/null +++ b/examples/success/input_directory_ftp05_merge.json @@ -0,0 +1,35 @@ +{ + "description": "And now two source directories merged to a common place. There should be 6 files altogether in /tes/old-tools", + "inputs": [ + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/das", + "path": "/tes/old-tools", + "type": "DIRECTORY" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/tools/dos", + "path": "/tes/old-tools", + "type": "DIRECTORY" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\; | sort -nr" + ], + "workdir": "/tes/old-tools" + }, + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type f" + ], + "workdir": "/tes/old-tools" + } + ] +} diff --git a/examples/success/input_file_directory_merge.json b/examples/success/input_file_directory_merge.json new file mode 100644 index 00000000..1b5c135d --- /dev/null +++ b/examples/success/input_file_directory_merge.json @@ -0,0 +1,25 @@ +{ + "description": "Demonstrates handling 2 http file inputs, that need to go to single directory.", + "inputs": [ + { + "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/examples/success/hello.json", + "path": "/tes/json", + "type": "FILE" + }, + { + "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/scripts/taskmaster.py", + "path": "/tes/python", + "type": "FILE" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find /tes -type f" + ] + } + ] +} diff --git a/examples/success/input_file_http.json b/examples/success/input_file_http.json new file mode 100644 index 00000000..53cb270c --- /dev/null +++ b/examples/success/input_file_http.json @@ -0,0 +1,19 @@ +{ + "description": "Demonstrates handling single http file input. Will output some nice looking JSON to stdout.", + "inputs": [ + { + "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/examples/success/hello.json", + "path": "/tes/volumes/input", + "type": "FILE" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "cat", + "/tes/volumes/input" + ] + } + ] +} diff --git a/examples/success/input_file_http_nested_mounts.json b/examples/success/input_file_http_nested_mounts.json new file mode 100644 index 00000000..7ed5834e --- /dev/null +++ b/examples/success/input_file_http_nested_mounts.json @@ -0,0 +1,25 @@ +{ + "description": "Demonstrates handling 2 http file inputs with nested mountpoints. Should find both files in their respective locations.", + "inputs": [ + { + "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/examples/success/hello.json", + "path": "/tes/volumes/input.json", + "type": "FILE" + }, + { + "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/examples/success/env.json", + "path": "/tes/file.json", + "type": "FILE" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -name '*.json'" + ] + } + ] +} From b553c55c29d5d03cbd823c85cd6c2609b97df995 Mon Sep 17 00:00:00 2001 From: aniewielska Date: Thu, 1 Mar 2018 14:39:18 +0000 Subject: [PATCH 5/9] Input examples --- examples/success/input_content.json | 19 +++++++++ .../input_dir_duplicate_file_names.json | 35 ++++++++++++++++ .../input_dir_file_duplicate_file_names.json | 35 ++++++++++++++++ ...01_tree.json => input_dir_ftp01_tree.json} | 0 ...tp02_sub.json => input_dir_ftp02_sub.json} | 0 ...on => input_dir_ftp03_tree_shadowing.json} | 0 ...on => input_dir_ftp04_tree_shadowing.json} | 0 ..._merge.json => input_dir_ftp05_merge.json} | 0 .../success/input_dir_ftp06_file_merge.json | 40 +++++++++++++++++++ ...y_merge.json => input_file_dir_merge.json} | 0 .../success/input_file_duplicate_names.json | 24 +++++++++++ 11 files changed, 153 insertions(+) create mode 100644 examples/success/input_content.json create mode 100644 examples/success/input_dir_duplicate_file_names.json create mode 100644 examples/success/input_dir_file_duplicate_file_names.json rename examples/success/{input_directory_ftp01_tree.json => input_dir_ftp01_tree.json} (100%) rename examples/success/{input_directory_ftp02_sub.json => input_dir_ftp02_sub.json} (100%) rename examples/success/{input_directory_ftp03_tree_shadowing.json => input_dir_ftp03_tree_shadowing.json} (100%) rename examples/success/{input_directory_ftp04_tree_shadowing.json => input_dir_ftp04_tree_shadowing.json} (100%) rename examples/success/{input_directory_ftp05_merge.json => input_dir_ftp05_merge.json} (100%) create mode 100644 examples/success/input_dir_ftp06_file_merge.json rename examples/success/{input_file_directory_merge.json => input_file_dir_merge.json} (100%) create mode 100644 examples/success/input_file_duplicate_names.json diff --git a/examples/success/input_content.json b/examples/success/input_content.json new file mode 100644 index 00000000..fbc209b7 --- /dev/null +++ b/examples/success/input_content.json @@ -0,0 +1,19 @@ +{ + "description": "Demonstrates inlined input", + "inputs": [ + { + "content": "ABC TESK and some more text.", + "path": "/tes/volumes/input", + "type": "FILE" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "cat", + "/tes/volumes/input" + ] + } + ] +} diff --git a/examples/success/input_dir_duplicate_file_names.json b/examples/success/input_dir_duplicate_file_names.json new file mode 100644 index 00000000..07bcbaaa --- /dev/null +++ b/examples/success/input_dir_duplicate_file_names.json @@ -0,0 +1,35 @@ +{ + "description": "2 different folders with filename collision, mounted to the same place. The latest one wins (das in our case)", + "inputs": [ + { + "url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA/README", + "path": "/tes/files/rna_README", + "type": "FILE" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/das/README", + "path": "/tes/files/das_README", + "type": "FILE" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA", + "path": "/tes", + "type": "DIRECTORY" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/das", + "path": "/tes", + "type": "DIRECTORY" + } + ], + "executors": [ + { + "image": "ubuntu", + "command": [ + "sh", + "-c", + "cd /tes; find * -name \"*README\" -print0 | du --files0-from=- -b" + ] + } + ] +} diff --git a/examples/success/input_dir_file_duplicate_file_names.json b/examples/success/input_dir_file_duplicate_file_names.json new file mode 100644 index 00000000..f55e6737 --- /dev/null +++ b/examples/success/input_dir_file_duplicate_file_names.json @@ -0,0 +1,35 @@ +{ + "description": "Placing a file at a location (/tes/README), that is colliding with a file from directory input. The latest one wins (das in our case)", + "inputs": [ + { + "url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA/README", + "path": "/tes/files/rna_README", + "type": "FILE" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/das/README", + "path": "/tes/files/das_README", + "type": "FILE" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA/README", + "path": "/tes/README", + "type": "FILE" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/das", + "path": "/tes", + "type": "DIRECTORY" + } + ], + "executors": [ + { + "image": "ubuntu", + "command": [ + "sh", + "-c", + "cd /tes; find * -name \"*README\" -print0 | du --files0-from=- -b" + ] + } + ] +} diff --git a/examples/success/input_directory_ftp01_tree.json b/examples/success/input_dir_ftp01_tree.json similarity index 100% rename from examples/success/input_directory_ftp01_tree.json rename to examples/success/input_dir_ftp01_tree.json diff --git a/examples/success/input_directory_ftp02_sub.json b/examples/success/input_dir_ftp02_sub.json similarity index 100% rename from examples/success/input_directory_ftp02_sub.json rename to examples/success/input_dir_ftp02_sub.json diff --git a/examples/success/input_directory_ftp03_tree_shadowing.json b/examples/success/input_dir_ftp03_tree_shadowing.json similarity index 100% rename from examples/success/input_directory_ftp03_tree_shadowing.json rename to examples/success/input_dir_ftp03_tree_shadowing.json diff --git a/examples/success/input_directory_ftp04_tree_shadowing.json b/examples/success/input_dir_ftp04_tree_shadowing.json similarity index 100% rename from examples/success/input_directory_ftp04_tree_shadowing.json rename to examples/success/input_dir_ftp04_tree_shadowing.json diff --git a/examples/success/input_directory_ftp05_merge.json b/examples/success/input_dir_ftp05_merge.json similarity index 100% rename from examples/success/input_directory_ftp05_merge.json rename to examples/success/input_dir_ftp05_merge.json diff --git a/examples/success/input_dir_ftp06_file_merge.json b/examples/success/input_dir_ftp06_file_merge.json new file mode 100644 index 00000000..d8684c0f --- /dev/null +++ b/examples/success/input_dir_ftp06_file_merge.json @@ -0,0 +1,40 @@ +{ + "description": "And now add files from different locations to directory tree retrieved from FTP. There should be 4 files in dos and 8 in mac.", + "inputs": [ + { + "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/README.md", + "path": "/tes/old-tools/mac/readme.md", + "type": "FILE" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/tools", + "path": "/tes/old-tools", + "type": "DIRECTORY" + }, + { + "url": "ftp://ftp.ebi.ac.uk/pub/software/das/README", + "path": "/tes/old-tools/dos/readme.txt", + "type": "FILE" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type d -exec sh -c 'echo \"$(find \"{}\" -type f | wc -l)\" {}' \\; | sort -nr" + ], + "workdir": "/tes/old-tools" + }, + { + "image": "alpine", + "command": [ + "sh", + "-c", + "find . -type f" + ], + "workdir": "/tes/old-tools" + } + ] +} diff --git a/examples/success/input_file_directory_merge.json b/examples/success/input_file_dir_merge.json similarity index 100% rename from examples/success/input_file_directory_merge.json rename to examples/success/input_file_dir_merge.json diff --git a/examples/success/input_file_duplicate_names.json b/examples/success/input_file_duplicate_names.json new file mode 100644 index 00000000..fb34e53d --- /dev/null +++ b/examples/success/input_file_duplicate_names.json @@ -0,0 +1,24 @@ +{ + "description": "Demonstrates an attempt of placing 2 different files in the same place. At the moment, teh last one wins (will overwrite previous occurrences)", + "inputs": [ + { + "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/examples/success/hello.json", + "path": "/tes/volumes/input", + "type": "FILE" + }, + { + "url": "https://raw.githubusercontent.com/EMBL-EBI-TSI/TESK/master/examples/success/stdin.json", + "path": "/tes/volumes/input", + "type": "FILE" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "cat", + "/tes/volumes/input" + ] + } + ] +} From 600260b63c26ff15c0bcb32e08aab4845b14c708 Mon Sep 17 00:00:00 2001 From: aniewielska Date: Thu, 1 Mar 2018 17:42:49 +0000 Subject: [PATCH 6/9] Example that can be used to test task cancel. --- examples/cancel/counter.json | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 examples/cancel/counter.json diff --git a/examples/cancel/counter.json b/examples/cancel/counter.json new file mode 100644 index 00000000..aa368290 --- /dev/null +++ b/examples/cancel/counter.json @@ -0,0 +1,23 @@ +{ + "description": "An example task to test job cancellation. Cancel it please, because it runs forever", + "volumes": [ + "/test" + ], + "executors": [ + { + "image": "alpine", + "command": [ + "echo", + "Nothing" + ] + }, + { + "image": "alpine", + "command": [ + "sh", + "-c", + "i=0; while true; do echo \"$i: $(date)\"; i=$((i+1)); sleep 1; done" + ] + } + ] +} From 7736e8a24d57136754d4ffad4e7824a5f5fc330a Mon Sep 17 00:00:00 2001 From: aniewielska Date: Fri, 2 Mar 2018 11:15:44 +0000 Subject: [PATCH 7/9] Outputs - draft --- examples/success/output.json | 47 +++++++++ examples/success/output_volume.json | 38 ++++++++ .../success/output_volume_taskmaster.json | 95 +++++++++++++++++++ 3 files changed, 180 insertions(+) create mode 100644 examples/success/output.json create mode 100644 examples/success/output_volume.json create mode 100644 examples/success/output_volume_taskmaster.json diff --git a/examples/success/output.json b/examples/success/output.json new file mode 100644 index 00000000..105214c5 --- /dev/null +++ b/examples/success/output.json @@ -0,0 +1,47 @@ +{ + "description": "FTP output", + "outputs": [ + { + "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing1 exists and is initially empty - expect file1.txt created", + "path": "/tes/file1.txt", + "type": "FILE", + "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing1/file1.txt" + }, + { + "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/new1 does not exist - expect directory will be created and will have file_new.txt inside with contents of file1.txt", + "path": "/tes/file1.txt", + "type": "FILE", + "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/new1/file_new.txt" + }, + { + "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/new2 does not exist - expect will be created and will contain file1.txt and file2.txt", + "path": "/tes", + "type": "DIRECTORY", + "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/new2" + }, + { + "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing2 exists and has file1.txt and remaining.txt inside - expect file1.txt will be replaced, remaining.txt untouched and file2.txt added", + "path": "/tes", + "type": "DIRECTORY", + "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing2" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "echo", + "This goes to file1" + ], + "stdout": "/tes/file1.txt" + }, + { + "image": "alpine", + "command": [ + "sh", + "-c", + "echo 'This goes to file2' > /tes/file2.txt" + ] + } + ] +} diff --git a/examples/success/output_volume.json b/examples/success/output_volume.json new file mode 100644 index 00000000..f2895e98 --- /dev/null +++ b/examples/success/output_volume.json @@ -0,0 +1,38 @@ +{ + "description": "FTP outputting file and directory from volume location - TODO fix expects in descriptions", + "volumes": [ + "/tes" + ], + "outputs": [ + { + "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing1 exists and is initially empty - expect file1.txt created", + "path": "/tes/file1.txt", + "type": "FILE", + "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing1/file1.txt" + }, + { + "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/new2 does not exist - expect will be created and will contain file1.txt and file2.txt", + "path": "/tes", + "type": "DIRECTORY", + "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/new2" + } + ], + "executors": [ + { + "image": "alpine", + "command": [ + "echo", + "This goes to file1" + ], + "stdout": "/tes/file1.txt" + }, + { + "image": "alpine", + "command": [ + "sh", + "-c", + "echo 'This goes to file2' > /tes/file2.txt" + ] + } + ] +} diff --git a/examples/success/output_volume_taskmaster.json b/examples/success/output_volume_taskmaster.json new file mode 100644 index 00000000..024f1633 --- /dev/null +++ b/examples/success/output_volume_taskmaster.json @@ -0,0 +1,95 @@ +{ + "outputs": [ + { + "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing1 exists and is initially empty - expect file1.txt created", + "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/existing1/file1.txt", + "path": "/tes/file1.txt", + "type": "FILE" + }, + { + "description": "ftp://ftp-private.ebi.ac.uk/upload/examples/new2 does not exist - expect will be created and will contain file1.txt and file2.txt", + "url": "ftp://ftp-private.ebi.ac.uk/upload/examples/new2", + "path": "/tes", + "type": "DIRECTORY" + } + ], + "inputs": [], + "volumes": [ + "/tes" + ], + "executors": [ + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-af78a6a3", + "executor-no": "0" + }, + "name": "task-af78a6a3-ex-00" + }, + "spec": { + "template": { + "metadata": { + "name": "task-af78a6a3-ex-00" + }, + "spec": { + "containers": [ + { + "command": [ + "/bin/sh", + "-c", + "echo \u0027This goes to file1\u0027 \u003e /tes/file1.txt" + ], + "image": "alpine", + "name": "task-af78a6a3-ex-00", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + }, + { + "apiVersion": "batch/v1", + "kind": "Job", + "metadata": { + "annotations": {}, + "labels": { + "job-type": "executor", + "taskmaster-name": "task-af78a6a3", + "executor-no": "1" + }, + "name": "task-af78a6a3-ex-01" + }, + "spec": { + "template": { + "metadata": { + "name": "task-af78a6a3-ex-01" + }, + "spec": { + "containers": [ + { + "command": [ + "sh", + "-c", + "echo \u0027This goes to file2\u0027 \u003e /tes/file2.txt" + ], + "image": "alpine", + "name": "task-af78a6a3-ex-01", + "resources": {} + } + ], + "restartPolicy": "Never" + } + } + } + } + ], + "resources": { + "disk_gb": 0.1 + } +} \ No newline at end of file From 7f1f1e6ee41754a0a88d05404cf7ec012266a913 Mon Sep 17 00:00:00 2001 From: Erik van den Bergh Date: Mon, 5 Mar 2018 10:05:48 +0000 Subject: [PATCH 8/9] Create tesintro.md --- tesintro.md | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 tesintro.md diff --git a/tesintro.md b/tesintro.md new file mode 100644 index 00000000..5e896b28 --- /dev/null +++ b/tesintro.md @@ -0,0 +1,33 @@ +# A very brief introduction to TES + +## Task Execution Service standard +The Global Alliance for Genomics and Health (GA4GH) is an international consortium of academic and industry partners that try to establish standards to promote and facilitate collaoration and data exchange in the life sciences. As part of the 'Cloud Workstream' of this effort 4 standards have been proposed to facilitate running scientific workflows in a cloud environment: the Data Object Service (DOS), Tool Registration Service (TRS), Workflow Execution Service (WES) and the Task Execution Service (TES). These for standards are meant to be independent but complementary to each other in running workflows and handling the associated data needs. TES is a standard that represents the smallest unit of computational work in a workflow that can be indepently run in a cloud. TESK is an implementation of this standard by EMBL-EBI running on Google's Kubernetes container orchestration platform. + +## Technical overview +A minimal TES task is represented as follows: + +```json +{ + "inputs": [ + { + "url": "http://adresss/to/input_file", + "path": "/container/input" + } + ], + "outputs" : [ + { + "url" : "file://path/to/output_file", + "path" : "/container/output" + } + ], + "executors" : [ + { + "image" : "ubuntu", + "command" : ["md5sum", "/container/input"], + "stdout" : "/container/output" + } + ] +} +``` + +Inputs and outputs are expected to have an URI that can be resolved by the relevant implementation. The executor 'image' entry is ay image that can be reached by the relevant docker instance of the implementation, and would usually refer to a public image on Dockerhub or Quay. TES tasks are submitted through a RESTful API using JSON. Also see the [full spec](https://github.com/ga4gh/task-execution-schemas) for a complete list of possible fields and their description. From dbdf45ed1a315e8bd48689347a1f83e90b516ada Mon Sep 17 00:00:00 2001 From: Erik van den Bergh Date: Mon, 5 Mar 2018 10:07:02 +0000 Subject: [PATCH 9/9] Add link to TES intro --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 13fa25b9..e3a7587c 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -An implementation of a task execution engine based on the [TES standard](https://github.com/ga4gh/task-execution-schemas) running on Kubernetes. +An implementation of a task execution engine based on the [TES standard](https://github.com/ga4gh/task-execution-schemas) running on Kubernetes. For more details on TES, see the [(very) brief introduction to TES](tesintro.md). For organisational reasons, this project is split into 2 repositories: One containing the API and associated docker images ([here](https://github.com/EMBL-EBI-TSI/tesk-api)) and one containing the actual task execution service and associated Docker images (this one). If the API is running on your cluster it will pull the images from our gcr.io repository automatically. In that vein, see below under 'How to install' to get TESK up and running on your Kubernetes cluster.