Skip to content

Commit 51424b9

Browse files
Merge pull request #43 from common-workflow-language/addToilBackend
Add a toil backend.
2 parents 6e960bf + f3c93b8 commit 51424b9

File tree

6 files changed

+360
-17
lines changed

6 files changed

+360
-17
lines changed

.travis.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,14 @@ python:
33
- '2.7'
44
before_install:
55
- sudo apt-get update -qq
6+
- pip install toil[all]==3.16.0
67
- pip install . --process-dependency-links
78
- pip install -r dev-requirements.txt
8-
- pip install toil[all]==3.16.0
99
script:
1010
- flake8 wes_service wes_client
1111
- pytest
12+
services:
13+
- docker
1214
deploy:
1315
provider: pypi
1416
on:

test/test_integration.py

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,13 @@
88
import signal
99
import requests
1010
import shutil
11+
import logging
12+
13+
logging.basicConfig(level=logging.INFO)
1114

1215

1316
class IntegrationTest(unittest.TestCase):
1417
"""A baseclass that's inherited for use with different cwl backends."""
15-
1618
def setUp(self):
1719
"""Start a (local) wes-service server to make requests against."""
1820
raise NotImplementedError
@@ -27,42 +29,40 @@ def tearDown(self):
2729
time.sleep(3)
2830
except OSError as e:
2931
print(e)
30-
32+
if os.path.exists('workflows'):
33+
shutil.rmtree('workflows')
3134
unittest.TestCase.tearDown(self)
3235

3336
def test_dockstore_md5sum(self):
3437
"""Fetch the md5sum cwl from dockstore, run it on the wes-service server, and check for the correct output."""
3538
cwl_dockstore_url = 'https://dockstore.org:8443/api/ga4gh/v2/tools/quay.io%2Fbriandoconnor%2Fdockstore-tool-md5sum/versions/master/plain-CWL/descriptor/%2FDockstore.cwl'
36-
output_filepath, _ = run_md5sum(cwl_input=cwl_dockstore_url)
39+
output_filepath, _ = run_cwl_md5sum(cwl_input=cwl_dockstore_url)
3740

3841
self.assertTrue(check_for_file(output_filepath), 'Output file was not found: ' + str(output_filepath))
39-
shutil.rmtree('workflows')
4042

4143
def test_local_md5sum(self):
4244
"""Pass a local md5sum cwl to the wes-service server, and check for the correct output."""
4345
cwl_local_path = os.path.abspath('testdata/md5sum.cwl')
44-
output_filepath, _ = run_md5sum(cwl_input='file://' + cwl_local_path)
46+
output_filepath, _ = run_cwl_md5sum(cwl_input='file://' + cwl_local_path)
4547

4648
self.assertTrue(check_for_file(output_filepath), 'Output file was not found: ' + str(output_filepath))
47-
shutil.rmtree('workflows')
4849

4950
def test_multipart_upload(self):
5051
"""Pass a local md5sum cwl to the wes-service server, and check for uploaded file in service."""
5152
cwl_local_path = os.path.abspath('testdata/md5sum.cwl')
52-
_, run_id = run_md5sum(cwl_input='file://' + cwl_local_path)
53+
_, run_id = run_cwl_md5sum(cwl_input='file://' + cwl_local_path)
5354

5455
get_response = get_log_request(run_id)["request"]
5556

5657
self.assertTrue(check_for_file(get_response["workflow_url"][7:]), 'Output file was not found: '
5758
+ get_response["workflow_url"][:7])
58-
shutil.rmtree('workflows')
5959

6060

61-
def run_md5sum(cwl_input):
61+
def run_cwl_md5sum(cwl_input):
6262
"""Pass a local md5sum cwl to the wes-service server, and return the path of the output file that was created."""
6363
endpoint = 'http://localhost:8080/ga4gh/wes/v1/runs'
6464
params = {'output_file': {'path': '/tmp/md5sum.txt', 'class': 'File'},
65-
'input_file': {'path': '../../testdata/md5sum.input', 'class': 'File'}}
65+
'input_file': {'path': os.path.abspath('testdata/md5sum.input'), 'class': 'File'}}
6666

6767
parts = [("workflow_params", json.dumps(params)), ("workflow_type", "CWL"), ("workflow_type_version", "v1.0")]
6868
if cwl_input.startswith("file://"):
@@ -75,6 +75,25 @@ def run_md5sum(cwl_input):
7575
return os.path.join(output_dir, 'md5sum.txt'), response['run_id']
7676

7777

78+
def run_wdl_md5sum(wdl_input):
79+
"""Pass a local md5sum wdl to the wes-service server, and return the path of the output file that was created."""
80+
endpoint = 'http://localhost:8080/ga4gh/wes/v1/workflows'
81+
params = '{"ga4ghMd5.inputFile": "' + os.path.abspath('testdata/md5sum.input') + '"}'
82+
parts = [("workflow_params", params),
83+
("workflow_type", "WDL"),
84+
("workflow_type_version", "v1.0"),
85+
("workflow_url", wdl_input)]
86+
response = requests.post(endpoint, files=parts).json()
87+
output_dir = os.path.abspath(os.path.join('workflows', response['workflow_id'], 'outdir'))
88+
check_travis_log = os.path.join(output_dir, 'stderr')
89+
with open(check_travis_log, 'r') as f:
90+
logging.info(f.read())
91+
logging.info(subprocess.check_output(['ls', os.path.join('workflows', response['workflow_id'])]))
92+
logging.info('\n')
93+
logging.info(subprocess.check_output(['ls', output_dir]))
94+
return os.path.join(output_dir, 'md5sum.txt'), response['workflow_id']
95+
96+
7897
def get_log_request(run_id):
7998
endpoint = 'http://localhost:8080/ga4gh/wes/v1/runs/{}'.format(run_id)
8099
return requests.get(endpoint).json()
@@ -88,7 +107,7 @@ def get_server_pids():
88107
return pids
89108

90109

91-
def check_for_file(filepath, seconds=20):
110+
def check_for_file(filepath, seconds=40):
92111
"""Return True if a file exists within a certain amount of time."""
93112
wait_counter = 0
94113
while not os.path.exists(filepath):
@@ -117,14 +136,12 @@ def setUp(self):
117136

118137
class ToilTest(IntegrationTest):
119138
"""Test using Toil."""
120-
121139
def setUp(self):
122140
"""
123141
Start a (local) wes-service server to make requests against.
124142
Use toil as the wes-service server 'backend'.
125143
"""
126-
self.wes_server_process = subprocess.Popen('python {} '
127-
'--opt runner=cwltoil --opt extra=--logLevel=CRITICAL'
144+
self.wes_server_process = subprocess.Popen('python {} --backend=wes_service.toil_wes --opt="extra=--logLevel=CRITICAL"'
128145
''.format(os.path.abspath('wes_service/wes_service_main.py')),
129146
shell=True)
130147
time.sleep(5)

testdata/md5sum.wdl

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
task md5 {
2+
File inputFile
3+
4+
command {
5+
/bin/my_md5sum ${inputFile}
6+
}
7+
8+
output {
9+
File value = "md5sum.txt"
10+
}
11+
12+
runtime {
13+
docker: "quay.io/briandoconnor/dockstore-tool-md5sum:1.0.4"
14+
cpu: 1
15+
memory: "512 MB"
16+
}
17+
}
18+
19+
workflow ga4ghMd5 {
20+
File inputFile
21+
call md5 { input: inputFile=inputFile }
22+
}

testdata/md5sum.wdl.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
{"ga4ghMd5.inputFile": "md5sum.input"}

wes_client/wes_client_main.py

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import logging
1111
import schema_salad.ref_resolver
1212
import requests
13-
from requests.exceptions import InvalidSchema
13+
from requests.exceptions import InvalidSchema, MissingSchema
1414
from wes_service.util import visit
1515
from bravado.client import SwaggerClient
1616
from bravado.requests_client import RequestsClient
@@ -81,6 +81,13 @@ def main(argv=sys.argv[1:]):
8181
json.dump(response.result(), sys.stdout, indent=4)
8282
return 0
8383

84+
if args.workflow_url.lower().endswith('wdl'):
85+
wf_type = 'WDL'
86+
elif args.workflow_url.lower().endswith('cwl'):
87+
wf_type = 'CWL'
88+
elif args.workflow_url.lower().endswith('py'):
89+
wf_type = 'PY'
90+
8491
if not args.job_order:
8592
logging.error("Missing job order")
8693
return 1
@@ -116,7 +123,7 @@ def fixpaths(d):
116123

117124
parts = [
118125
("workflow_params", json.dumps(input_dict)),
119-
("workflow_type", "CWL"),
126+
("workflow_type", wf_type),
120127
("workflow_type_version", "v1.0")
121128
]
122129
if workflow_url.startswith("file://"):
@@ -167,6 +174,8 @@ def fixpaths(d):
167174
logging.info("Workflow log:\n" + logs)
168175
except InvalidSchema:
169176
logging.info("Workflow log:\n" + str(s["workflow_log"]["stderr"]))
177+
except MissingSchema:
178+
logging.info("Workflow log:\n" + str(s["workflow_log"]["stderr"]))
170179

171180
if "fields" in s["outputs"] and s["outputs"]["fields"] is None:
172181
del s["outputs"]["fields"]

0 commit comments

Comments
 (0)