Skip to content

Commit 182a68c

Browse files
mr-cmr-c
committed
types for everyone
1 parent 654708d commit 182a68c

29 files changed

+339
-324
lines changed

‎cwltool/command_line_tool.py

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ def __init__(
106106
self,
107107
builder: Builder,
108108
script: str,
109-
output_callback: OutputCallbackType,
109+
output_callback: Optional[OutputCallbackType],
110110
requirements: List[CWLObjectType],
111111
hints: List[CWLObjectType],
112112
outdir: Optional[str] = None,
@@ -131,21 +131,23 @@ def run(
131131
normalizeFilesDirs(self.builder.job)
132132
ev = self.builder.do_eval(self.script)
133133
normalizeFilesDirs(ev)
134-
self.output_callback(ev, "success")
134+
if self.output_callback:
135+
self.output_callback(ev, "success")
135136
except WorkflowException as err:
136137
_logger.warning(
137138
"Failed to evaluate expression:\n%s",
138139
str(err),
139140
exc_info=runtimeContext.debug,
140141
)
141-
self.output_callback({}, "permanentFail")
142+
if self.output_callback:
143+
self.output_callback({}, "permanentFail")
142144

143145

144146
class ExpressionTool(Process):
145147
def job(
146148
self,
147149
job_order: CWLObjectType,
148-
output_callbacks: OutputCallbackType,
150+
output_callbacks: Optional[OutputCallbackType],
149151
runtimeContext: RuntimeContext,
150152
) -> Generator[ExpressionJob, None, None]:
151153
builder = self._init_job(job_order, runtimeContext)
@@ -165,7 +167,7 @@ class AbstractOperation(Process):
165167
def job(
166168
self,
167169
job_order: CWLObjectType,
168-
output_callbacks: OutputCallbackType,
170+
output_callbacks: Optional[OutputCallbackType],
169171
runtimeContext: RuntimeContext,
170172
) -> JobsGeneratorType:
171173
raise WorkflowException("Abstract operation cannot be executed.")
@@ -250,7 +252,7 @@ class CallbackJob(object):
250252
def __init__(
251253
self,
252254
job: "CommandLineTool",
253-
output_callback: OutputCallbackType,
255+
output_callback: Optional[OutputCallbackType],
254256
cachebuilder: Builder,
255257
jobcache: str,
256258
) -> None:
@@ -266,14 +268,15 @@ def run(
266268
runtimeContext: RuntimeContext,
267269
tmpdir_lock: Optional[threading.Lock] = None,
268270
) -> None:
269-
self.output_callback(
270-
self.job.collect_output_ports(
271-
self.job.tool["outputs"],
272-
self.cachebuilder,
273-
self.outdir,
274-
getdefault(runtimeContext.compute_checksum, True),
275-
),
276-
"success",
271+
if self.output_callback:
272+
self.output_callback(
273+
self.job.collect_output_ports(
274+
self.job.tool["outputs"],
275+
self.cachebuilder,
276+
self.outdir,
277+
getdefault(runtimeContext.compute_checksum, True),
278+
),
279+
"success",
277280
)
278281

279282

@@ -413,7 +416,7 @@ def updatePathmap(
413416
def job(
414417
self,
415418
job_order: CWLObjectType,
416-
output_callbacks: OutputCallbackType,
419+
output_callbacks: Optional[OutputCallbackType],
417420
runtimeContext: RuntimeContext,
418421
) -> Generator[Union[JobBase, CallbackJob], None, None]:
419422

‎cwltool/job.py

Lines changed: 32 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
from .process import stage_files
4949
from .secrets import SecretStore
5050
from .utils import (
51+
OutputCallbackType,
5152
DEFAULT_TMP_PREFIX,
5253
CWLObjectType,
5354
Directory,
@@ -223,7 +224,7 @@ def __init__(
223224
self.collect_outputs = cast(
224225
Callable[[str, int], MutableMapping[str, Any]], None
225226
) # type: Union[Callable[[str, int], MutableMapping[str, Any]], functools.partial[MutableMapping[str, Any]]]
226-
self.output_callback = cast(Callable[[Any, Any], Any], None)
227+
self.output_callback = None # type: Optional[OutputCallbackType]
227228
self.outdir = ""
228229
self.tmpdir = ""
229230

@@ -291,7 +292,7 @@ def _execute(
291292
runtime: List[str],
292293
env: MutableMapping[str, str],
293294
runtimeContext: RuntimeContext,
294-
monitor_function=None, # type: Optional[Callable[[subprocess.Popen[str]], None]]
295+
monitor_function = None # type: Optional[Callable[[subprocess.Popen[str]], None]]
295296
) -> None:
296297

297298
scr = self.get_requirement("ShellCommandRequirement")[0]
@@ -468,8 +469,9 @@ def _execute(
468469
"runtimeContext.workflow_eval_lock must not be None"
469470
)
470471

471-
with runtimeContext.workflow_eval_lock:
472-
self.output_callback(outputs, processStatus)
472+
if self.output_callback:
473+
with runtimeContext.workflow_eval_lock:
474+
self.output_callback(outputs, processStatus)
473475

474476
if self.stagedir is not None and os.path.exists(self.stagedir):
475477
_logger.debug(
@@ -486,8 +488,8 @@ def _execute(
486488
shutil.rmtree(self.tmpdir, True)
487489

488490
def process_monitor(
489-
self, sproc # type: subprocess.Popen[str]
490-
) -> None:
491+
self, sproc
492+
): # type: (subprocess.Popen[str]) -> None
491493
monitor = psutil.Process(sproc.pid)
492494
# Value must be list rather than integer to utilise pass-by-reference in python
493495
memory_usage = [None] # type: MutableSequence[Optional[int]]
@@ -525,7 +527,7 @@ def run(
525527
self,
526528
runtimeContext: RuntimeContext,
527529
tmpdir_lock: Optional[threading.Lock] = None,
528-
): # type: (...) -> None
530+
) -> None:
529531

530532
if tmpdir_lock:
531533
with tmpdir_lock:
@@ -601,16 +603,15 @@ def get_from_requirements(
601603
@abstractmethod
602604
def create_runtime(
603605
self,
604-
env, # type: MutableMapping[str, str]
605-
runtime_context, # type: RuntimeContext
606-
): # type: (...) -> Tuple[List[str], Optional[str]]
606+
env: MutableMapping[str, str],
607+
runtime_context: RuntimeContext,
608+
) -> Tuple[List[str], Optional[str]]:
607609
"""Return the list of commands to run the selected container engine."""
608610
pass
609611

610612
@staticmethod
611613
@abstractmethod
612-
def append_volume(runtime, source, target, writable=False):
613-
# type: (List[str], str, str, bool) -> None
614+
def append_volume(runtime: List[str], source: str, target: str, writable: bool=False) -> None:
614615
"""Add binding arguments to the runtime list."""
615616
pass
616617

@@ -624,22 +625,22 @@ def add_file_or_directory_volume(
624625
@abstractmethod
625626
def add_writable_file_volume(
626627
self,
627-
runtime, # type: List[str]
628-
volume, # type: MapperEnt
629-
host_outdir_tgt, # type: Optional[str]
630-
tmpdir_prefix, # type: str
631-
): # type: (...) -> None
628+
runtime: List[str],
629+
volume: MapperEnt,
630+
host_outdir_tgt: Optional[str],
631+
tmpdir_prefix: str,
632+
) -> None:
632633
"""Append a writable file mapping to the runtime option list."""
633634
pass
634635

635636
@abstractmethod
636637
def add_writable_directory_volume(
637638
self,
638-
runtime, # type: List[str]
639-
volume, # type: MapperEnt
640-
host_outdir_tgt, # type: Optional[str]
641-
tmpdir_prefix, # type: str
642-
): # type: (...) -> None
639+
runtime: List[str],
640+
volume: MapperEnt,
641+
host_outdir_tgt: Optional[str],
642+
tmpdir_prefix: str,
643+
) -> None:
643644
"""Append a writable directory mapping to the runtime option list."""
644645
pass
645646

@@ -678,12 +679,12 @@ def create_file_and_add_volume(
678679

679680
def add_volumes(
680681
self,
681-
pathmapper, # type: PathMapper
682-
runtime, # type: List[str]
683-
tmpdir_prefix, # type: str
684-
secret_store=None, # type: Optional[SecretStore]
685-
any_path_okay=False, # type: bool
686-
): # type: (...) -> None
682+
pathmapper: PathMapper,
683+
runtime: List[str],
684+
tmpdir_prefix: str,
685+
secret_store: Optional[SecretStore] = None,
686+
any_path_okay: bool = False,
687+
) -> None:
687688
"""Append volume mappings to the runtime option list."""
688689
container_outdir = self.builder.outdir
689690
for key, vol in (itm for itm in pathmapper.items() if itm[1].staged):
@@ -716,9 +717,9 @@ def add_volumes(
716717

717718
def run(
718719
self,
719-
runtimeContext, # type: RuntimeContext
720-
tmpdir_lock=None, # type: Optional[threading.Lock]
721-
): # type: (...) -> None
720+
runtimeContext: RuntimeContext,
721+
tmpdir_lock: Optional[threading.Lock] = None,
722+
) -> None:
722723
if tmpdir_lock:
723724
with tmpdir_lock:
724725
if not os.path.exists(self.tmpdir):

‎cwltool/main.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -534,20 +534,20 @@ def remove_non_cwl(deps): # type: (MutableMapping[str, Any]) -> None
534534

535535

536536
def find_deps(
537-
obj, # type: Mapping[str, Any]
538-
document_loader, # type: Loader
539-
uri, # type: str
540-
basedir=None, # type: Optional[str]
541-
nestdirs=True, # type: bool
542-
): # type: (...) -> Dict[str, Any]
537+
obj: Mapping[str, Any],
538+
document_loader: Loader,
539+
uri: str,
540+
basedir: Optional[str] = None,
541+
nestdirs: bool =True,
542+
) -> Dict[str, Any]:
543543
"""Find the dependencies of the CWL document."""
544544
deps = {
545545
"class": "File",
546546
"location": uri,
547547
"format": CWL_IANA,
548548
} # type: Dict[str, Any]
549549

550-
def loadref(base, uri): # type: (str, str) -> Any
550+
def loadref(base: str, uri: str) -> Union[CommentedMap, CommentedSeq, str, None]:
551551
return document_loader.fetch(document_loader.fetcher.urljoin(base, uri))
552552

553553
sfs = scandeps(

‎cwltool/process.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -995,7 +995,7 @@ def visit(self, op: Callable[[CommentedMap], None]) -> None:
995995
def job(
996996
self,
997997
job_order: CWLObjectType,
998-
output_callbacks: OutputCallbackType,
998+
output_callbacks: Optional[OutputCallbackType],
999999
runtimeContext: RuntimeContext,
10001000
) -> JobsGeneratorType:
10011001
pass
@@ -1073,7 +1073,7 @@ def scandeps(
10731073
doc: Any,
10741074
reffields: Set[str],
10751075
urlfields: Set[str],
1076-
loadref: Callable[[str, str], str],
1076+
loadref: Callable[[str, str], Union[CommentedMap, CommentedSeq, str, None]],
10771077
urljoin: Callable[[str, str], str] = urllib.parse.urljoin,
10781078
nestdirs: bool = True,
10791079
) -> List[Dict[str, str]]:

‎cwltool/procgenerator.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ def receive_output(
4242
def job(
4343
self,
4444
job_order: CWLObjectType,
45-
output_callbacks: OutputCallbackType,
45+
output_callbacks: Optional[OutputCallbackType],
4646
runtimeContext: RuntimeContext,
4747
) -> JobsGeneratorType:
4848

@@ -55,7 +55,7 @@ def job(
5555
while self.processStatus is None:
5656
yield None
5757

58-
if self.processStatus != "success":
58+
if self.processStatus != "success" and output_callbacks:
5959
output_callbacks(self.jobout, self.processStatus)
6060
return
6161

@@ -101,7 +101,7 @@ def __init__(
101101
def job(
102102
self,
103103
job_order: CWLObjectType,
104-
output_callbacks: OutputCallbackType,
104+
output_callbacks: Optional[OutputCallbackType],
105105
runtimeContext: RuntimeContext,
106106
) -> JobsGeneratorType:
107107
return ProcessGeneratorJob(self).job(

‎cwltool/provenance.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@
3333
)
3434

3535
import prov.model as provM
36-
from pathlib2 import Path, PurePath, PurePosixPath
36+
from pathlib import Path, PurePath, PurePosixPath
3737
from prov.identifier import Identifier, Namespace
3838
from prov.model import PROV, ProvDocument, ProvEntity
3939
from ruamel import yaml

‎cwltool/resolver.py

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,13 @@
55
import urllib
66
from typing import Any, Optional
77

8-
from pathlib2 import Path
8+
from pathlib import Path
99
from schema_salad.ref_resolver import Loader
1010

1111
from .loghandler import _logger
1212

1313

14-
def resolve_local(document_loader, uri):
15-
# type: (Loader, str) -> Optional[str]
14+
def resolve_local(document_loader: Optional[Loader], uri: str) -> Optional[str]:
1615
pathpart, frag = urllib.parse.urldefrag(uri)
1716

1817
try:

‎cwltool/utils.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
from cachecontrol import CacheControl
3939
from cachecontrol.caches import FileCache
4040
from mypy_extensions import TypedDict
41-
from pathlib2 import Path
41+
from pathlib import Path
4242
from schema_salad.exceptions import ValidationException
4343
from schema_salad.ref_resolver import Loader
4444
from typing_extensions import TYPE_CHECKING, Deque
@@ -95,7 +95,7 @@
9595
]
9696
JobsGeneratorType = Generator[JobsType, None, None]
9797
OutputCallbackType = Callable[[Optional[CWLObjectType], str], None]
98-
ResolverType = Callable[["Loader", str], str]
98+
ResolverType = Callable[["Loader", str], Optional[str]]
9999
DestinationsType = MutableMapping[str, Optional[CWLOutputType]]
100100
ScatterDestinationsType = MutableMapping[str, List[Optional[CWLOutputType]]]
101101
ScatterOutputCallbackType = Callable[[Optional[ScatterDestinationsType], str], None]

0 commit comments

Comments
 (0)