Skip to content

Commit 654708d

Browse files
mr-cmr-c
committed
even more
1 parent e382302 commit 654708d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

57 files changed

+1631
-232
lines changed

‎cwltool/executors.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -431,6 +431,7 @@ def run_jobs(
431431

432432
class NoopJobExecutor(JobExecutor):
433433
""" Do nothing executor, for testing purposes only. """
434+
434435
def run_jobs(
435436
self,
436437
process: Process,
@@ -446,5 +447,5 @@ def execute(
446447
job_order_object: Dict[str, Any],
447448
runtime_context: RuntimeContext,
448449
logger: Optional[logging.Logger] = None,
449-
) -> Tuple[Union[Optional[CWLObjectType], MutableSequence[CWLObjectType]], str]:
450+
) -> Tuple[Union[Optional[CWLObjectType], MutableSequence[CWLObjectType]], str]:
450451
return {}, "success"

‎cwltool/job.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,11 @@
3434

3535
import psutil
3636
import shellescape
37+
from prov.model import PROV
3738
from schema_salad.sourceline import SourceLine
3839
from schema_salad.utils import json_dump, json_dumps
3940
from typing_extensions import TYPE_CHECKING
4041

41-
from prov.model import PROV
42-
4342
from .builder import Builder, HasReqsHints
4443
from .context import RuntimeContext, getdefault
4544
from .errors import UnsupportedRequirement, WorkflowException
@@ -491,9 +490,9 @@ def process_monitor(
491490
) -> None:
492491
monitor = psutil.Process(sproc.pid)
493492
# Value must be list rather than integer to utilise pass-by-reference in python
494-
memory_usage = [None]
493+
memory_usage = [None] # type: MutableSequence[Optional[int]]
495494

496-
def get_tree_mem_usage(memory_usage): # type: (List[int]) -> None
495+
def get_tree_mem_usage(memory_usage: MutableSequence[Optional[int]]) -> None:
497496
children = monitor.children()
498497
rss = monitor.memory_info().rss
499498
while len(children):

‎cwltool/provenance.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,16 +32,15 @@
3232
cast,
3333
)
3434

35+
import prov.model as provM
3536
from pathlib2 import Path, PurePath, PurePosixPath
37+
from prov.identifier import Identifier, Namespace
38+
from prov.model import PROV, ProvDocument, ProvEntity
3639
from ruamel import yaml
3740
from schema_salad.sourceline import SourceLine
3841
from schema_salad.utils import json_dumps
3942
from typing_extensions import TYPE_CHECKING
4043

41-
import prov.model as provM
42-
from prov.identifier import Identifier, Namespace
43-
from prov.model import PROV, ProvDocument, ProvEntity
44-
4544
from .context import RuntimeContext
4645
from .errors import WorkflowException
4746
from .loghandler import _logger
@@ -167,6 +166,7 @@ def __init__(self, research_object: "ResearchObject", rel_path: str) -> None:
167166
)
168167
if not path.startswith(os.path.abspath(research_object.folder)):
169168
raise ValueError("Path is outside Research Object: %s" % path)
169+
_logger.debug("[provenance] Creating WritableBagFile at %s.", path)
170170
super(WritableBagFile, self).__init__(path, mode="w")
171171

172172
def write(self, b):
@@ -181,8 +181,8 @@ def write(self, b):
181181
ret = super(WritableBagFile, self).write(real_b)
182182
if ret:
183183
total += ret
184-
for _ in self.hashes.values():
185-
_.update(real_b)
184+
for val in self.hashes.values():
185+
val.update(real_b)
186186
return total
187187

188188
def close(self) -> None:

‎setup.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
"cwltool/cwlrdf.py",
3737
"cwltool/docker_id.py",
3838
"cwltool/docker.py",
39+
"cwltool/udocker.py",
3940
"cwltool/errors.py",
4041
"cwltool/executors.py",
4142
"cwltool/expression.py",

‎tests/test_anon_types.py

Lines changed: 81 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -7,104 +7,109 @@
77
from cwltool.command_line_tool import CommandLineTool
88
from cwltool.context import LoadingContext
99

10-
snippet = cast(CommentedMap, cmap([
11-
{
12-
"cwlVersion": "v1.0",
13-
"class": "CommandLineTool",
14-
"inputs": [
10+
snippet = cast(
11+
CommentedMap,
12+
cmap(
13+
[
1514
{
16-
"type": {
17-
"type": "record",
18-
"fields": [
19-
{
20-
"type": [
15+
"cwlVersion": "v1.0",
16+
"class": "CommandLineTool",
17+
"inputs": [
18+
{
19+
"type": {
20+
"type": "record",
21+
"fields": [
2122
{
22-
"type": "enum",
23-
"symbols": [
24-
"anon_enum_inside_array.cwl#first/species/homo_sapiens",
25-
"anon_enum_inside_array.cwl#first/species/mus_musculus",
23+
"type": [
24+
{
25+
"type": "enum",
26+
"symbols": [
27+
"anon_enum_inside_array.cwl#first/species/homo_sapiens",
28+
"anon_enum_inside_array.cwl#first/species/mus_musculus",
29+
],
30+
},
31+
"null",
2632
],
27-
},
28-
"null",
33+
"name": "anon_enum_inside_array.cwl#first/species",
34+
}
2935
],
30-
"name": "anon_enum_inside_array.cwl#first/species",
31-
}
32-
],
33-
},
34-
"id": "anon_enum_inside_array.cwl#first",
35-
},
36-
{
37-
"type": [
38-
"null",
36+
},
37+
"id": "anon_enum_inside_array.cwl#first",
38+
},
3939
{
40-
"type": "enum",
41-
"symbols": [
42-
"anon_enum_inside_array.cwl#second/homo_sapiens",
43-
"anon_enum_inside_array.cwl#second/mus_musculus",
40+
"type": [
41+
"null",
42+
{
43+
"type": "enum",
44+
"symbols": [
45+
"anon_enum_inside_array.cwl#second/homo_sapiens",
46+
"anon_enum_inside_array.cwl#second/mus_musculus",
47+
],
48+
},
4449
],
50+
"id": "anon_enum_inside_array.cwl#second",
4551
},
4652
],
47-
"id": "anon_enum_inside_array.cwl#second",
53+
"baseCommand": "echo",
54+
"outputs": [],
55+
"id": "anon_enum_inside_array.cwl",
4856
},
49-
],
50-
"baseCommand": "echo",
51-
"outputs": [],
52-
"id": "anon_enum_inside_array.cwl",
53-
},
54-
{
55-
"cwlVersion": "v1.0",
56-
"class": "CommandLineTool",
57-
"requirements": [
5857
{
59-
"types": [
58+
"cwlVersion": "v1.0",
59+
"class": "CommandLineTool",
60+
"requirements": [
6061
{
61-
"name": "anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params",
62-
"type": "record",
63-
"fields": [
62+
"types": [
6463
{
65-
"type": [
66-
"null",
64+
"name": "anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params",
65+
"type": "record",
66+
"fields": [
6767
{
68-
"type": "enum",
69-
"symbols": [
70-
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/ncbi_build/GRCh37",
71-
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/ncbi_build/GRCh38",
72-
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/ncbi_build/GRCm38",
68+
"type": [
69+
"null",
70+
{
71+
"type": "enum",
72+
"symbols": [
73+
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/ncbi_build/GRCh37",
74+
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/ncbi_build/GRCh38",
75+
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/ncbi_build/GRCm38",
76+
],
77+
},
7378
],
79+
"name": "anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/ncbi_build",
7480
},
75-
],
76-
"name": "anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/ncbi_build",
77-
},
78-
{
79-
"type": [
80-
"null",
8181
{
82-
"type": "enum",
83-
"symbols": [
84-
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/species/homo_sapiens",
85-
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/species/mus_musculus",
82+
"type": [
83+
"null",
84+
{
85+
"type": "enum",
86+
"symbols": [
87+
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/species/homo_sapiens",
88+
"anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/species/mus_musculus",
89+
],
90+
},
8691
],
92+
"name": "anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/species",
8793
},
8894
],
89-
"name": "anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params/species",
90-
},
95+
}
9196
],
97+
"class": "SchemaDefRequirement",
9298
}
9399
],
94-
"class": "SchemaDefRequirement",
95-
}
96-
],
97-
"inputs": [
98-
{
99-
"type": "anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params",
100-
"id": "anon_enum_inside_array_inside_schemadef.cwl#first",
101-
}
102-
],
103-
"baseCommand": "echo",
104-
"outputs": [],
105-
"id": "anon_enum_inside_array_inside_schemadef.cwl",
106-
},
107-
]))
100+
"inputs": [
101+
{
102+
"type": "anon_enum_inside_array_inside_schemadef.cwl#vcf2maf_params",
103+
"id": "anon_enum_inside_array_inside_schemadef.cwl#first",
104+
}
105+
],
106+
"baseCommand": "echo",
107+
"outputs": [],
108+
"id": "anon_enum_inside_array_inside_schemadef.cwl",
109+
},
110+
]
111+
),
112+
)
108113

109114

110115
@pytest.mark.parametrize("snippet", snippet) # type: ignore

‎tests/test_pack.py

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from io import StringIO
66
from tempfile import NamedTemporaryFile
77

8-
import pytest
8+
import pytest # type: ignore
99
from ruamel import yaml
1010

1111
import cwltool.pack
@@ -20,7 +20,7 @@
2020
from .util import get_data, needs_docker
2121

2222

23-
def test_pack():
23+
def test_pack() -> None:
2424
loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/revsort.cwl"))
2525

2626
with open(get_data("tests/wf/expect_packed.cwl")) as packed_file:
@@ -40,15 +40,17 @@ def test_pack():
4040
assert packed == expect_packed
4141

4242

43-
def test_pack_input_named_name():
43+
def test_pack_input_named_name() -> None:
4444
loadingContext, workflowobj, uri = fetch_document(
4545
get_data("tests/wf/trick_revsort.cwl")
4646
)
4747
loadingContext.do_update = False
4848
loadingContext, uri = resolve_and_validate_document(
4949
loadingContext, workflowobj, uri
5050
)
51-
processobj = loadingContext.loader.resolve_ref(uri)[0]
51+
loader = loadingContext.loader
52+
assert loader
53+
processobj = loader.resolve_ref(uri)[0]
5254

5355
with open(get_data("tests/wf/expect_trick_packed.cwl")) as packed_file:
5456
expect_packed = yaml.round_trip_load(packed_file)
@@ -67,7 +69,7 @@ def test_pack_input_named_name():
6769
assert packed == expect_packed
6870

6971

70-
def test_pack_single_tool():
72+
def test_pack_single_tool() -> None:
7173
loadingContext, workflowobj, uri = fetch_document(
7274
get_data("tests/wf/formattest.cwl")
7375
)
@@ -81,7 +83,7 @@ def test_pack_single_tool():
8183
assert "$schemas" in packed
8284

8385

84-
def test_pack_fragment():
86+
def test_pack_fragment() -> None:
8587
with open(get_data("tests/wf/scatter2_subwf.cwl")) as packed_file:
8688
expect_packed = yaml.safe_load(packed_file)
8789

@@ -97,7 +99,7 @@ def test_pack_fragment():
9799
)
98100

99101

100-
def test_pack_rewrites():
102+
def test_pack_rewrites() -> None:
101103
rewrites = {}
102104

103105
loadingContext, workflowobj, uri = fetch_document(
@@ -122,8 +124,8 @@ def test_pack_rewrites():
122124
]
123125

124126

125-
@pytest.mark.parametrize("cwl_path", cwl_missing_version_paths)
126-
def test_pack_missing_cwlVersion(cwl_path):
127+
@pytest.mark.parametrize("cwl_path", cwl_missing_version_paths) # type: ignore
128+
def test_pack_missing_cwlVersion(cwl_path: str) -> None:
127129
"""Ensure the generated pack output is not missing the `cwlVersion` in case of single tool workflow and single step workflow."""
128130
# Testing single tool workflow
129131
loadingContext, workflowobj, uri = fetch_document(get_data(cwl_path))
@@ -139,17 +141,17 @@ def test_pack_missing_cwlVersion(cwl_path):
139141
assert packed["cwlVersion"] == "v1.0"
140142

141143

142-
def test_pack_idempotence_tool():
144+
def test_pack_idempotence_tool() -> None:
143145
"""Ensure that pack produces exactly the same document for an already packed CommandLineTool."""
144146
_pack_idempotently("tests/wf/hello_single_tool.cwl")
145147

146148

147-
def test_pack_idempotence_workflow():
149+
def test_pack_idempotence_workflow() -> None:
148150
"""Ensure that pack produces exactly the same document for an already packed workflow."""
149151
_pack_idempotently("tests/wf/count-lines1-wf.cwl")
150152

151153

152-
def _pack_idempotently(document):
154+
def _pack_idempotently(document: str) -> None:
153155
loadingContext, workflowobj, uri = fetch_document(get_data(document))
154156
loadingContext.do_update = False
155157
loadingContext, uri = resolve_and_validate_document(
@@ -190,9 +192,11 @@ def _pack_idempotently(document):
190192
]
191193

192194

193-
@needs_docker
194-
@pytest.mark.parametrize("wf_path,job_path,namespaced", cwl_to_run)
195-
def test_packed_workflow_execution(wf_path, job_path, namespaced, tmpdir):
195+
@needs_docker # type: ignore
196+
@pytest.mark.parametrize("wf_path,job_path,namespaced", cwl_to_run) # type: ignore
197+
def test_packed_workflow_execution(
198+
wf_path: str, job_path: str, namespaced: bool, tmpdir: py.path.local
199+
) -> None:
196200
loadingContext = LoadingContext()
197201
loadingContext.resolver = tool_resolver
198202
loadingContext, workflowobj, uri = fetch_document(get_data(wf_path), loadingContext)

0 commit comments

Comments
 (0)