Skip to content

Commit e6c2d95

Browse files
mr-cmr-c
authored
Merge pull request #1299 from common-workflow-language/less_any
Rework types to use "Any" less often
2 parents e8b3565 + 024317b commit e6c2d95

35 files changed

+2918
-2537
lines changed

‎Makefile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,10 @@ mypy: ${PYSOURCES}
160160
MYPYPATH=$$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
161161
--warn-redundant-casts \
162162
cwltool
163+
164+
mypyc: ${PYSOURCES}
165+
MYPYPATH=typeshed/2and3/:typeshed/3 CWLTOOL_USE_MYPYC=1 pip install --verbose -e . && pytest --ignore cwltool/schemas --basetemp ./tmp
166+
163167
release-test: FORCE
164168
git diff-index --quiet HEAD -- || ( echo You have uncommited changes, please commit them and try again; false )
165169
./release-test.sh

‎cwltool/builder.py

Lines changed: 68 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
from .errors import WorkflowException
3131
from .loghandler import _logger
3232
from .mutation import MutationManager
33+
from .software_requirements import DependenciesConfiguration
3334
from .stdfsaccess import StdFsAccess
3435
from .utils import (
3536
CONTENT_LIMIT,
@@ -44,7 +45,7 @@
4445
)
4546

4647
if TYPE_CHECKING:
47-
from .provenance import ProvenanceProfile # pylint: disable=unused-import
48+
from .provenance_profile import ProvenanceProfile # pylint: disable=unused-import
4849
from .pathmapper import PathMapper
4950

5051

@@ -108,10 +109,10 @@ def formatSubclassOf(
108109

109110

110111
def check_format(
111-
actual_file, # type: Union[Dict[str, Any], List[Dict[str, Any]], str]
112-
input_formats, # type: Union[List[str], str]
113-
ontology, # type: Optional[Graph]
114-
): # type: (...) -> None
112+
actual_file: Union[CWLObjectType, List[CWLObjectType]],
113+
input_formats: Union[List[str], str],
114+
ontology: Optional[Graph],
115+
) -> None:
115116
"""Confirm that the format present is valid for the allowed formats."""
116117
for afile in aslist(actual_file):
117118
if not afile:
@@ -154,16 +155,16 @@ def __init__(
154155
job: CWLObjectType,
155156
files: List[CWLObjectType],
156157
bindings: List[CWLObjectType],
157-
schemaDefs: Dict[str, CWLObjectType],
158+
schemaDefs: MutableMapping[str, CWLObjectType],
158159
names: Names,
159160
requirements: List[CWLObjectType],
160161
hints: List[CWLObjectType],
161-
resources: Dict[str, int],
162+
resources: Dict[str, Union[int, float]],
162163
mutation_manager: Optional[MutationManager],
163164
formatgraph: Optional[Graph],
164165
make_fs_access: Type[StdFsAccess],
165166
fs_access: StdFsAccess,
166-
job_script_provider: Optional[Any],
167+
job_script_provider: Optional[DependenciesConfiguration],
167168
timeout: float,
168169
debug: bool,
169170
js_console: bool,
@@ -208,21 +209,18 @@ def __init__(
208209
self.find_default_container = None # type: Optional[Callable[[], str]]
209210

210211
def build_job_script(self, commands: List[str]) -> Optional[str]:
211-
build_job_script_method = getattr(
212-
self.job_script_provider, "build_job_script", None
213-
) # type: Optional[Callable[[Builder, Union[List[str],List[str]]], str]]
214-
if build_job_script_method is not None:
215-
return build_job_script_method(self, commands)
212+
if self.job_script_provider is not None:
213+
return self.job_script_provider.build_job_script(self, commands)
216214
return None
217215

218216
def bind_input(
219217
self,
220-
schema: MutableMapping[str, Any],
221-
datum: Any,
218+
schema: CWLObjectType,
219+
datum: Union[CWLObjectType, List[CWLObjectType]],
222220
discover_secondaryFiles: bool,
223221
lead_pos: Optional[Union[int, List[int]]] = None,
224222
tail_pos: Optional[Union[str, List[int]]] = None,
225-
) -> List[MutableMapping[str, Any]]:
223+
) -> List[MutableMapping[str, Union[str, List[int]]]]:
226224

227225
if tail_pos is None:
228226
tail_pos = []
@@ -267,9 +265,9 @@ def bind_input(
267265
elif (
268266
isinstance(t, MutableMapping)
269267
and "name" in t
270-
and self.names.has_name(t["name"], None)
268+
and self.names.has_name(cast(str, t["name"]), None)
271269
):
272-
avsc = self.names.get_name(t["name"], None)
270+
avsc = self.names.get_name(cast(str, t["name"]), None)
273271
if not avsc:
274272
avsc = make_avsc_object(convert_to_dict(t), self.names)
275273
if validate(avsc, datum):
@@ -329,30 +327,35 @@ def bind_input(
329327
)
330328
else:
331329
if schema["type"] in self.schemaDefs:
332-
schema = self.schemaDefs[schema["type"]]
330+
schema = self.schemaDefs[cast(str, schema["type"])]
333331

334332
if schema["type"] == "record":
335-
for f in schema["fields"]:
336-
if f["name"] in datum and datum[f["name"]] is not None:
333+
datum = cast(CWLObjectType, datum)
334+
for f in cast(List[CWLObjectType], schema["fields"]):
335+
name = cast(str, f["name"])
336+
if name in datum and datum[name] is not None:
337337
bindings.extend(
338338
self.bind_input(
339339
f,
340-
datum[f["name"]],
340+
cast(CWLObjectType, datum[name]),
341341
lead_pos=lead_pos,
342-
tail_pos=f["name"],
342+
tail_pos=name,
343343
discover_secondaryFiles=discover_secondaryFiles,
344344
)
345345
)
346346
else:
347-
datum[f["name"]] = f.get("default")
347+
datum[name] = f.get("default")
348348

349349
if schema["type"] == "array":
350-
for n, item in enumerate(datum):
350+
for n, item in enumerate(cast(MutableSequence[CWLObjectType], datum)):
351351
b2 = None
352352
if binding:
353-
b2 = copy.deepcopy(binding)
353+
b2 = cast(CWLObjectType, copy.deepcopy(binding))
354354
b2["datum"] = item
355-
itemschema = {"type": schema["items"], "inputBinding": b2}
355+
itemschema = {
356+
"type": schema["items"],
357+
"inputBinding": b2,
358+
} # type: CWLObjectType
356359
for k in ("secondaryFiles", "format", "streamable"):
357360
if k in schema:
358361
itemschema[k] = schema[k]
@@ -367,17 +370,18 @@ def bind_input(
367370
)
368371
binding = {}
369372

370-
def _capture_files(f): # type: (CWLObjectType) -> CWLObjectType
373+
def _capture_files(f: CWLObjectType) -> CWLObjectType:
371374
self.files.append(f)
372375
return f
373376

374377
if schema["type"] == "File":
378+
datum = cast(CWLObjectType, datum)
375379
self.files.append(datum)
376380
if (binding and binding.get("loadContents")) or schema.get(
377381
"loadContents"
378382
):
379-
with self.fs_access.open(datum["location"], "rb") as f:
380-
datum["contents"] = content_limit_respected_read(f)
383+
with self.fs_access.open(cast(str, datum["location"]), "rb") as f2:
384+
datum["contents"] = content_limit_respected_read(f2)
381385

382386
if "secondaryFiles" in schema:
383387
if "secondaryFiles" not in datum:
@@ -391,7 +395,9 @@ def _capture_files(f): # type: (CWLObjectType) -> CWLObjectType
391395
if "$(" in sf["pattern"] or "${" in sf["pattern"]:
392396
sfpath = self.do_eval(sf["pattern"], context=datum)
393397
else:
394-
sfpath = substitute(datum["basename"], sf["pattern"])
398+
sfpath = substitute(
399+
cast(str, datum["basename"]), sf["pattern"]
400+
)
395401

396402
for sfname in aslist(sfpath):
397403
if not sfname:
@@ -400,8 +406,8 @@ def _capture_files(f): # type: (CWLObjectType) -> CWLObjectType
400406

401407
if isinstance(sfname, str):
402408
sf_location = (
403-
datum["location"][
404-
0 : datum["location"].rindex("/") + 1
409+
cast(str, datum["location"])[
410+
0 : cast(str, datum["location"]).rindex("/") + 1
405411
]
406412
+ sfname
407413
)
@@ -415,7 +421,10 @@ def _capture_files(f): # type: (CWLObjectType) -> CWLObjectType
415421
% (type(sfname))
416422
)
417423

418-
for d in datum["secondaryFiles"]:
424+
for d in cast(
425+
MutableSequence[MutableMapping[str, str]],
426+
datum["secondaryFiles"],
427+
):
419428
if not d.get("basename"):
420429
d["basename"] = d["location"][
421430
d["location"].rindex("/") + 1 :
@@ -426,8 +435,8 @@ def _capture_files(f): # type: (CWLObjectType) -> CWLObjectType
426435
if not found:
427436

428437
def addsf(
429-
files: MutableSequence[MutableMapping[str, Any]],
430-
newsf: MutableMapping[str, Any],
438+
files: MutableSequence[CWLObjectType],
439+
newsf: CWLObjectType,
431440
) -> None:
432441
for f in files:
433442
if f["location"] == newsf["location"]:
@@ -436,12 +445,21 @@ def addsf(
436445
files.append(newsf)
437446

438447
if isinstance(sfname, MutableMapping):
439-
addsf(datum["secondaryFiles"], sfname)
448+
addsf(
449+
cast(
450+
MutableSequence[CWLObjectType],
451+
datum["secondaryFiles"],
452+
),
453+
sfname,
454+
)
440455
elif discover_secondaryFiles and self.fs_access.exists(
441456
sf_location
442457
):
443458
addsf(
444-
datum["secondaryFiles"],
459+
cast(
460+
MutableSequence[CWLObjectType],
461+
datum["secondaryFiles"],
462+
),
445463
{
446464
"location": sf_location,
447465
"basename": sfname,
@@ -454,12 +472,16 @@ def addsf(
454472
% (sfname, json_dumps(datum, indent=4))
455473
)
456474

457-
normalizeFilesDirs(datum["secondaryFiles"])
475+
normalizeFilesDirs(
476+
cast(MutableSequence[CWLObjectType], datum["secondaryFiles"])
477+
)
458478

459479
if "format" in schema:
460480
try:
461481
check_format(
462-
datum, self.do_eval(schema["format"]), self.formatgraph
482+
datum,
483+
cast(Union[List[str], str], self.do_eval(schema["format"])),
484+
self.formatgraph,
463485
)
464486
except ValidationException as ve:
465487
raise WorkflowException(
@@ -474,9 +496,12 @@ def addsf(
474496
)
475497

476498
if schema["type"] == "Directory":
499+
datum = cast(CWLObjectType, datum)
477500
ll = schema.get("loadListing") or self.loadListing
478501
if ll and ll != "no_listing":
479-
get_listing(self.fs_access, datum, (ll == "deep_listing"))
502+
get_listing(
503+
self.fs_access, datum, (ll == "deep_listing"),
504+
)
480505
self.files.append(datum)
481506

482507
if schema["type"] == "Any":
@@ -576,13 +601,11 @@ def generate_arg(self, binding: CWLObjectType) -> List[str]:
576601

577602
def do_eval(
578603
self,
579-
ex: Union[
580-
str, float, bool, None, MutableMapping[str, str], MutableSequence[str]
581-
],
604+
ex: Optional[CWLOutputType],
582605
context: Optional[Any] = None,
583606
recursive: bool = False,
584607
strip_whitespace: bool = True,
585-
) -> Any:
608+
) -> Optional[CWLOutputType]:
586609
if recursive:
587610
if isinstance(ex, MutableMapping):
588611
return {k: self.do_eval(v, context, recursive) for k, v in ex.items()}

0 commit comments

Comments
 (0)