Merge branch 'master' into testing

elixir-cloud-aai · Mar 5, 2018 · ee659e5 · ee659e5
2 parents 580037c + dbdf45e
commit ee659e5
Show file tree

Hide file tree

Showing 27 changed files with 909 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -1,7 +1,7 @@
 <img src="docs/TESKlogowfont.png" height="200">
 
 
-An implementation of a task execution engine based on the [TES standard](https://github.com/ga4gh/task-execution-schemas) running on Kubernetes. 
+An implementation of a task execution engine based on the [TES standard](https://github.com/ga4gh/task-execution-schemas) running on Kubernetes. For more details on TES, see the [(very) brief introduction to TES](tesintro.md).
 
 For organisational reasons, this project is split into 2 repositories: One containing the API and associated docker images ([here](https://github.com/EMBL-EBI-TSI/tesk-api)) and one containing the actual task execution service and associated Docker images (this one). If the API is running on your cluster it will pull the images from our gcr.io repository automatically. In that vein, see below under 'How to install' to get TESK up and running on your Kubernetes cluster.
 
@@ -10,7 +10,8 @@ For organisational reasons, this project is split into 2 repositories: One conta
 
 ## How to install
  - Clone the repo to your kube-master and cd into the folder
- - Edit the following line in `specs/ingress/nginx-ingress-lb.yaml`:
+ - Find out what is the external IP for the cluster. E.g. with the command `minikube ip`
+ - Edit the following line in `specs/ingress/nginx-ingress-lb.yaml` with that IP:
 
  ```yaml
  spec:
@@ -20,7 +21,7 @@ For organisational reasons, this project is split into 2 repositories: One conta
 
  ```
 
- - Then create the services necessary to run the API:
+ - Create the services necessary to run the API:
 
 ```
 kubectl create -f specs/ingress/

diff --git a/examples/cancel/counter.json b/examples/cancel/counter.json
@@ -0,0 +1,23 @@
+{
+  "description": "An example task to test job cancellation. Cancel it please, because it runs forever",
+  "volumes": [
+    "/test"
+  ],
+  "executors": [
+    {
+      "image": "alpine",
+      "command": [
+        "echo",
+        "Nothing"
+      ]
+    },
+    {
+      "image": "alpine",
+      "command": [
+        "sh",
+        "-c",
+        "i=0; while true; do echo \"$i: $(date)\"; i=$((i+1)); sleep 1; done"
+      ]
+    }
+  ]
+}
diff --git a/examples/error/error_stops_execution_taskmaster.json b/examples/error/error_stops_execution_taskmaster.json
@@ -0,0 +1,113 @@
+{
+  "outputs": [],
+  "inputs": [],
+  "volumes": [],
+  "executors": [
+    {
+      "apiVersion": "batch/v1",
+      "kind": "Job",
+      "metadata": {
+        "annotations": {},
+        "labels": {
+          "job-type": "executor",
+          "taskmaster-name": "task-ec104c85",
+          "executor-no": "0"
+        },
+        "name": "task-ec104c85-ex-00"
+      },
+      "spec": {
+        "template": {
+          "metadata": {
+            "name": "task-ec104c85-ex-00"
+          },
+          "spec": {
+            "containers": [
+              {
+                "command": [
+                  "echo",
+                  "You will see this in the logs (stdout)."
+                ],
+                "image": "alpine",
+                "name": "task-ec104c85-ex-00",
+                "resources": {}
+              }
+            ],
+            "restartPolicy": "Never"
+          }
+        }
+      }
+    },
+    {
+      "apiVersion": "batch/v1",
+      "kind": "Job",
+      "metadata": {
+        "annotations": {},
+        "labels": {
+          "job-type": "executor",
+          "taskmaster-name": "task-ec104c85",
+          "executor-no": "1"
+        },
+        "name": "task-ec104c85-ex-01"
+      },
+      "spec": {
+        "template": {
+          "metadata": {
+            "name": "task-ec104c85-ex-01"
+          },
+          "spec": {
+            "containers": [
+              {
+                "command": [
+                  "sh",
+                  "-c",
+                  "exit 1"
+                ],
+                "image": "alpine",
+                "name": "task-ec104c85-ex-01",
+                "resources": {}
+              }
+            ],
+            "restartPolicy": "Never"
+          }
+        }
+      }
+    },
+    {
+      "apiVersion": "batch/v1",
+      "kind": "Job",
+      "metadata": {
+        "annotations": {},
+        "labels": {
+          "job-type": "executor",
+          "taskmaster-name": "task-ec104c85",
+          "executor-no": "2"
+        },
+        "name": "task-ec104c85-ex-02"
+      },
+      "spec": {
+        "template": {
+          "metadata": {
+            "name": "task-ec104c85-ex-02"
+          },
+          "spec": {
+            "containers": [
+              {
+                "command": [
+                  "echo",
+                  "This shouldn't appear in the logs (stdout)."
+                ],
+                "image": "alpine",
+                "name": "task-ec104c85-ex-02",
+                "resources": {}
+              }
+            ],
+            "restartPolicy": "Never"
+          }
+        }
+      }
+    }
+  ],
+  "resources": {
+    "disk_gb": 0.1
+  }
+}
diff --git a/examples/error/executor_taskmaster.json b/examples/error/executor_taskmaster.json
@@ -0,0 +1,45 @@
+{
+  "outputs": [],
+  "inputs": [],
+  "volumes": [],
+  "executors": [
+    {
+      "apiVersion": "batch/v1",
+      "kind": "Job",
+      "metadata": {
+        "annotations": {},
+        "labels": {
+          "job-type": "executor",
+          "taskmaster-name": "task-23bf5e64",
+          "executor-no": "0"
+        },
+        "name": "task-23bf5e64-ex-00"
+      },
+      "spec": {
+        "template": {
+          "metadata": {
+            "name": "task-23bf5e64-ex-00"
+          },
+          "spec": {
+            "containers": [
+              {
+                "command": [
+                  "sh",
+                  "-c",
+                  "exit 1"
+                ],
+                "image": "alpine",
+                "name": "task-23bf5e64-ex-00",
+                "resources": {}
+              }
+            ],
+            "restartPolicy": "Never"
+          }
+        }
+      }
+    }
+  ],
+  "resources": {
+    "disk_gb": 0.1
+  }
+}
diff --git a/examples/error/input_taskmaster.json b/examples/error/input_taskmaster.json
@@ -0,0 +1,84 @@
+{
+  "outputs": [],
+  "inputs": [
+    {
+      "url": "http://nonexistent.ebi.ac.uk",
+      "path": "/somewhere/file",
+      "type": "FILE"
+    }
+  ],
+  "volumes": [],
+  "executors": [
+    {
+      "apiVersion": "batch/v1",
+      "kind": "Job",
+      "metadata": {
+        "annotations": {},
+        "labels": {
+          "job-type": "executor",
+          "taskmaster-name": "task-7ff304bd",
+          "executor-no": "0"
+        },
+        "name": "task-7ff304bd-ex-00"
+      },
+      "spec": {
+        "template": {
+          "metadata": {
+            "name": "task-7ff304bd-ex-00"
+          },
+          "spec": {
+            "containers": [
+              {
+                "command": [
+                  "echo",
+                  "This shouldn't appear in the logs."
+                ],
+                "image": "alpine",
+                "name": "task-7ff304bd-ex-00",
+                "resources": {}
+              }
+            ],
+            "restartPolicy": "Never"
+          }
+        }
+      }
+    },
+    {
+      "apiVersion": "batch/v1",
+      "kind": "Job",
+      "metadata": {
+        "annotations": {},
+        "labels": {
+          "job-type": "executor",
+          "taskmaster-name": "task-7ff304bd",
+          "executor-no": "1"
+        },
+        "name": "task-7ff304bd-ex-01"
+      },
+      "spec": {
+        "template": {
+          "metadata": {
+            "name": "task-7ff304bd-ex-01"
+          },
+          "spec": {
+            "containers": [
+              {
+                "command": [
+                  "cat",
+                  "/somewhere/file"
+                ],
+                "image": "alpine",
+                "name": "task-7ff304bd-ex-01",
+                "resources": {}
+              }
+            ],
+            "restartPolicy": "Never"
+          }
+        }
+      }
+    }
+  ],
+  "resources": {
+    "disk_gb": 0.1
+  }
+}
diff --git a/examples/success/input_content.json b/examples/success/input_content.json
@@ -0,0 +1,19 @@
+{
+  "description": "Demonstrates inlined input",
+  "inputs": [
+    {
+      "content": "ABC TESK and some more text.",
+      "path": "/tes/volumes/input",
+      "type": "FILE"
+    }
+  ],
+  "executors": [
+    {
+      "image": "alpine",
+      "command": [
+        "cat",
+        "/tes/volumes/input"
+      ]
+    }
+  ]
+}
diff --git a/examples/success/input_dir_duplicate_file_names.json b/examples/success/input_dir_duplicate_file_names.json
@@ -0,0 +1,35 @@
+{
+  "description": "2 different folders with filename collision, mounted to the same place. The latest one wins (das in our case)",
+  "inputs": [
+    {
+      "url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA/README",
+      "path": "/tes/files/rna_README",
+      "type": "FILE"
+    },
+    {
+      "url": "ftp://ftp.ebi.ac.uk/pub/software/das/README",
+      "path": "/tes/files/das_README",
+      "type": "FILE"
+    },
+    {
+      "url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA",
+      "path": "/tes",
+      "type": "DIRECTORY"
+    },
+    {
+      "url": "ftp://ftp.ebi.ac.uk/pub/software/das",
+      "path": "/tes",
+      "type": "DIRECTORY"
+    }
+  ],
+  "executors": [
+    {
+      "image": "ubuntu",
+      "command": [
+        "sh",
+        "-c",
+        "cd /tes;  find * -name \"*README\" -print0 | du --files0-from=- -b"
+      ]
+    }
+  ]
+}
diff --git a/examples/success/input_dir_file_duplicate_file_names.json b/examples/success/input_dir_file_duplicate_file_names.json
@@ -0,0 +1,35 @@
+{
+  "description": "Placing a file at a location (/tes/README), that is colliding with a file from directory input. The latest one wins (das in our case)",
+  "inputs": [
+    {
+      "url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA/README",
+      "path": "/tes/files/rna_README",
+      "type": "FILE"
+    },
+    {
+      "url": "ftp://ftp.ebi.ac.uk/pub/software/das/README",
+      "path": "/tes/files/das_README",
+      "type": "FILE"
+    },
+    {
+      "url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA/README",
+      "path": "/tes/README",
+      "type": "FILE"
+    },
+    {
+      "url": "ftp://ftp.ebi.ac.uk/pub/software/das",
+      "path": "/tes",
+      "type": "DIRECTORY"
+    }
+  ],
+  "executors": [
+    {
+      "image": "ubuntu",
+      "command": [
+        "sh",
+        "-c",
+        "cd /tes;  find * -name \"*README\" -print0 | du --files0-from=- -b"
+      ]
+    }
+  ]
+}