the tests pass!

Author: mr-c

cwltool/command_line_tool.py

@@ -277,7 +277,7 @@ def run(
                     getdefault(runtimeContext.compute_checksum, True),
                 ),
                 "success",
-        )
+            )
 
 
 def check_adjust(builder: Builder, file_o: CWLObjectType) -> CWLObjectType:

cwltool/context.py

@@ -118,7 +118,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
         self.basedir = ""  # type: str
         self.toplevel = False  # type: bool
         self.mutation_manager = None  # type: Optional[MutationManager]
-        self.make_fs_access = StdFsAccess  # type: Type[StdFsAccess]
+        self.make_fs_access = StdFsAccess  # type: Callable[[str], StdFsAccess]
         self.path_mapper = PathMapper
         self.builder = None  # type: Optional[Builder]
         self.docker_outdir = ""  # type: str

cwltool/executors.py

@@ -18,13 +18,14 @@
     Set,
     Tuple,
     Union,
+    cast,
 )
 
 import psutil
 from schema_salad.exceptions import ValidationException
 from schema_salad.sourceline import SourceLine
 
-from .command_line_tool import CallbackJob
+from .command_line_tool import CallbackJob, ExpressionJob
 from .context import RuntimeContext, getdefault
 from .errors import WorkflowException
 from .job import JobBase
@@ -63,19 +64,19 @@ def output_callback(
     def run_jobs(
         self,
         process: Process,
-        job_order_object: Dict[str, Any],
+        job_order_object: CWLObjectType,
         logger: logging.Logger,
         runtime_context: RuntimeContext,
     ) -> None:
         """Execute the jobs for the given Process."""
 
     def execute(
         self,
-        process,  # type: Process
-        job_order_object,  # type: Dict[str, Any]
-        runtime_context,  # type: RuntimeContext
-        logger=_logger,  # type: logging.Logger
-    ):  # type: (...) -> Tuple[Union[Optional[CWLObjectType], MutableSequence[CWLObjectType]], str]
+        process: Process,
+        job_order_object: CWLObjectType,
+        runtime_context: RuntimeContext,
+        logger: logging.Logger = _logger,
+    ) -> Tuple[Union[Optional[CWLObjectType], MutableSequence[CWLObjectType]], str]:
         """Execute the process."""
         if not runtime_context.basedir:
             raise WorkflowException("Must provide 'basedir' in runtimeContext")
@@ -102,7 +103,7 @@ def check_for_abstract_op(tool: MutableMapping[str, Any]) -> None:
         runtime_context.toplevel = True
         runtime_context.workflow_eval_lock = threading.Condition(threading.RLock())
 
-        job_reqs = None
+        job_reqs = None  # type: Optional[List[CWLObjectType]]
         if "https://w3id.org/cwl/cwl#requirements" in job_order_object:
             if (
                 process.metadata.get("http://commonwl.org/cwltool#original_cwlVersion")
@@ -113,7 +114,10 @@ def check_for_abstract_op(tool: MutableMapping[str, Any]) -> None:
                     "v1.0. You can adjust to use `cwltool:overrides` instead; or you "
                     "can set the cwlVersion to v1.1"
                 )
-            job_reqs = job_order_object["https://w3id.org/cwl/cwl#requirements"]
+            job_reqs = cast(
+                List[CWLObjectType],
+                job_order_object["https://w3id.org/cwl/cwl#requirements"],
+            )
         elif (
             "cwl:defaults" in process.metadata
             and "https://w3id.org/cwl/cwl#requirements"
@@ -299,7 +303,7 @@ def select_resources(
         return result
 
     def _runner(self, job, runtime_context, TMPDIR_LOCK):
-        # type: (Union[JobBase, WorkflowJob, CallbackJob], RuntimeContext, threading.Lock) -> None
+        # type: (Union[JobBase, WorkflowJob, CallbackJob, ExpressionJob], RuntimeContext, threading.Lock) -> None
         """Job running thread."""
         try:
             _logger.debug(
@@ -435,7 +439,7 @@ class NoopJobExecutor(JobExecutor):
     def run_jobs(
         self,
         process: Process,
-        job_order_object: Dict[str, Any],
+        job_order_object: CWLObjectType,
         logger: logging.Logger,
         runtime_context: RuntimeContext,
     ) -> None:
@@ -444,7 +448,7 @@ def run_jobs(
     def execute(
         self,
         process: Process,
-        job_order_object: Dict[str, Any],
+        job_order_object: CWLObjectType,
         runtime_context: RuntimeContext,
         logger: Optional[logging.Logger] = None,
     ) -> Tuple[Union[Optional[CWLObjectType], MutableSequence[CWLObjectType]], str]:

cwltool/job.py

@@ -34,11 +34,12 @@
 
 import psutil
 import shellescape
-from prov.model import PROV
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dump, json_dumps
 from typing_extensions import TYPE_CHECKING
 
+from prov.model import PROV
+
 from .builder import Builder, HasReqsHints
 from .context import RuntimeContext, getdefault
 from .errors import UnsupportedRequirement, WorkflowException
@@ -48,10 +49,10 @@
 from .process import stage_files
 from .secrets import SecretStore
 from .utils import (
-    OutputCallbackType,
     DEFAULT_TMP_PREFIX,
     CWLObjectType,
     Directory,
+    OutputCallbackType,
     bytes2str_in_dicts,
     copytree_with_merge,
     ensure_non_writable,
@@ -292,7 +293,7 @@ def _execute(
         runtime: List[str],
         env: MutableMapping[str, str],
         runtimeContext: RuntimeContext,
-        monitor_function = None  # type: Optional[Callable[[subprocess.Popen[str]], None]]
+        monitor_function=None,  # type: Optional[Callable[[subprocess.Popen[str]], None]]
     ) -> None:
 
         scr = self.get_requirement("ShellCommandRequirement")[0]
@@ -487,9 +488,7 @@ def _execute(
             )
             shutil.rmtree(self.tmpdir, True)
 
-    def process_monitor(
-        self, sproc
-        ): # type: (subprocess.Popen[str]) -> None
+    def process_monitor(self, sproc):  # type: (subprocess.Popen[str]) -> None
         monitor = psutil.Process(sproc.pid)
         # Value must be list rather than integer to utilise pass-by-reference in python
         memory_usage = [None]  # type: MutableSequence[Optional[int]]
@@ -602,16 +601,16 @@ def get_from_requirements(
 
     @abstractmethod
     def create_runtime(
-        self,
-        env: MutableMapping[str, str], 
-        runtime_context: RuntimeContext,
+        self, env: MutableMapping[str, str], runtime_context: RuntimeContext,
     ) -> Tuple[List[str], Optional[str]]:
         """Return the list of commands to run the selected container engine."""
         pass
 
     @staticmethod
     @abstractmethod
-    def append_volume(runtime: List[str], source: str, target: str, writable: bool=False) -> None:
+    def append_volume(
+        runtime: List[str], source: str, target: str, writable: bool = False
+    ) -> None:
         """Add binding arguments to the runtime list."""
         pass
 
@@ -719,7 +718,7 @@ def run(
         self,
         runtimeContext: RuntimeContext,
         tmpdir_lock: Optional[threading.Lock] = None,
-     ) -> None:
+    ) -> None:
         if tmpdir_lock:
             with tmpdir_lock:
                 if not os.path.exists(self.tmpdir):

cwltool/main.py

@@ -538,7 +538,7 @@ def find_deps(
     document_loader: Loader,
     uri: str,
     basedir: Optional[str] = None,
-    nestdirs: bool =True,
+    nestdirs: bool = True,
 ) -> Dict[str, Any]:
     """Find the dependencies of the CWL document."""
     deps = {

cwltool/provenance.py

@@ -13,6 +13,7 @@
 from collections import OrderedDict
 from getpass import getuser
 from io import BytesIO, FileIO, TextIOWrapper, open
+from pathlib import Path, PurePath, PurePosixPath
 from socket import getfqdn
 from types import ModuleType
 from typing import (
@@ -32,15 +33,15 @@
     cast,
 )
 
-import prov.model as provM
-from pathlib import Path, PurePath, PurePosixPath
-from prov.identifier import Identifier, Namespace
-from prov.model import PROV, ProvDocument, ProvEntity
 from ruamel import yaml
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dumps
 from typing_extensions import TYPE_CHECKING
 
+import prov.model as provM
+from prov.identifier import Identifier, Namespace
+from prov.model import PROV, ProvDocument, ProvEntity
+
 from .context import RuntimeContext
 from .errors import WorkflowException
 from .loghandler import _logger
@@ -169,19 +170,16 @@ def __init__(self, research_object: "ResearchObject", rel_path: str) -> None:
         _logger.debug("[provenance] Creating WritableBagFile at %s.", path)
         super(WritableBagFile, self).__init__(path, mode="w")
 
-    def write(self, b):
-        # type: (Union[bytes, str]) -> int
-        if isinstance(b, bytes):
-            real_b = b
-        else:
-            real_b = b.encode("utf-8")
+    def write(self, b: Union[bytes, str]) -> int:
+        real_b = b if isinstance(b, bytes) else b.encode("utf-8")
         total = 0
         length = len(real_b)
         while total < length:
             ret = super(WritableBagFile, self).write(real_b)
             if ret:
                 total += ret
         for val in self.hashes.values():
+            # print("Updating hasher %s ", val)
             val.update(real_b)
         return total
 

cwltool/resolver.py

@@ -3,9 +3,9 @@
 import os
 import sys
 import urllib
+from pathlib import Path
 from typing import Any, Optional
 
-from pathlib import Path
 from schema_salad.ref_resolver import Loader
 
 from .loghandler import _logger

cwltool/utils.py

@@ -14,6 +14,7 @@
 import uuid
 from functools import partial
 from itertools import zip_longest
+from pathlib import Path
 from tempfile import NamedTemporaryFile
 from types import ModuleType
 from typing import (
@@ -38,7 +39,6 @@
 from cachecontrol import CacheControl
 from cachecontrol.caches import FileCache
 from mypy_extensions import TypedDict
-from pathlib import Path
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import Loader
 from typing_extensions import TYPE_CHECKING, Deque

setup.py

@@ -53,7 +53,7 @@
         # "cwltool/pathmapper.py",  # class PathMapper needs to be subclassable
         "cwltool/process.py",
         "cwltool/procgenerator.py",
-        "cwltool/provenance.py",
+        # "cwltool/provenance.py",  # WritableBag is having issues
         "cwltool/resolver.py",
         # "cwltool/sandboxjs.py",  # probably not speed critical, tests need to mock components
         "cwltool/secrets.py",

tests/test_docker.py

@@ -1,10 +1,12 @@
 from distutils import spawn
 
+import py.path
+
 from cwltool.docker import DockerCommandLineJob
 from cwltool.main import main
 
 from .util import get_data, get_main_output, needs_docker, needs_singularity
-import py.path
+
 
 @needs_docker  # type: ignore
 def test_docker_workflow(tmpdir: py.path.local) -> None:

tests/test_docker_info.py

@@ -1,4 +1,3 @@
-
 from .util import get_data, get_main_output, needs_docker
 
 

tests/test_docker_paths_with_colons.py

@@ -1,9 +1,10 @@
+from typing import Any
+
 from cwltool.docker import DockerCommandLineJob
 from cwltool.main import main
 
 from .util import needs_docker
 
-from typing import Any
 
 def test_docker_append_volume_read_only(mocker: Any) -> None:
     mocker.patch("os.mkdir")