Merge pull request #1072 from common-workflow-language/inputBinding.position-expr

Input binding.position expr

Author: mr-c

cwltool/builder.py

@@ -211,7 +211,13 @@ def bind_input(self,
 
             bp = list(aslist(lead_pos))
             if "position" in binding:
-                bp.extend(aslist(binding["position"]))
+                position = binding["position"]
+                if isinstance(position, str):  # no need to test the CWL Version
+                                               # the schema for v1.0 only allow ints
+                    binding['position'] = self.do_eval(position, context=datum)
+                    bp.append(binding['position'])
+                else:
+                    bp.extend(aslist(binding['position']))
             else:
                 bp.append(0)
             bp.extend(aslist(tail_pos))

cwltool/command_line_tool.py

@@ -592,12 +592,17 @@ def collect_output_ports(self,
                              ports,                  # type: Set[Dict[Text, Any]]
                              builder,                # type: Builder
                              outdir,                 # type: Text
+                             rcode,                  # type: int
                              compute_checksum=True,  # type: bool
                              jobname="",             # type: Text
                              readers=None            # type: Dict[Text, Any]
                             ):  # type: (...) -> OutputPorts
         ret = {}  # type: OutputPorts
         debug = _logger.isEnabledFor(logging.DEBUG)
+        cwl_version = self.metadata.get(
+            "http://commonwl.org/cwltool#original_cwlVersion", None)
+        if cwl_version != "v1.0":
+            builder.resources["exitCode"] = rcode
         try:
             fs_access = builder.make_fs_access(outdir)
             custom_output = fs_access.join(outdir, "cwl.output.json")

cwltool/job.py

@@ -187,8 +187,8 @@ def __init__(self,
         self.generatemapper = None  # type: Optional[PathMapper]
 
         # set in CommandLineTool.job(i)
-        self.collect_outputs = cast(Callable[[Any], Any],
-                                    None)  # type: Union[Callable[[Any], Any], functools.partial[Any]]
+        self.collect_outputs = cast(Callable[[Text, int], MutableMapping[Text, Any]],
+                                    None)  # type: Union[Callable[[Text, int], MutableMapping[Text, Any]], functools.partial[MutableMapping[Text, Any]]]
         self.output_callback = cast(Callable[[Any, Any], Any], None)
         self.outdir = u""
         self.tmpdir = u""
@@ -261,7 +261,7 @@ def _execute(self,
             assert runtimeContext.prov_obj is not None
             runtimeContext.prov_obj.used_artefacts(
                 job_order, runtimeContext.process_run_id, str(self.name))
-        outputs = {}  # type: Dict[Text,Text]
+        outputs = {}  # type: MutableMapping[Text,Any]
         try:
             stdin_path = None
             if self.stdin is not None:
@@ -328,7 +328,7 @@ def _execute(self,
                     self.generatemapper, self.outdir, self.builder.outdir,
                     inplace_update=self.inplace_update)
 
-            outputs = self.collect_outputs(self.outdir)
+            outputs = self.collect_outputs(self.outdir, rcode)
             outputs = bytes2str_in_dicts(outputs)  # type: ignore
         except OSError as e:
             if e.errno == 2:
@@ -689,7 +689,7 @@ def docker_monitor(self, cidfile, tmpdir_prefix, cleanup_cidfile, process):
             try:
                 with open(cidfile) as cidhandle:
                     cid = cidhandle.readline().strip()
-            except OSError:
+            except (OSError, IOError):
                 cid = None
         max_mem = self.docker_get_memory(cid)
         tmp_dir, tmp_prefix = os.path.split(tmpdir_prefix)

cwltool/process.py

@@ -13,7 +13,7 @@
 import tempfile
 import textwrap
 import uuid
-from collections import Iterable  # pylint: disable=unused-import
+
 from io import open
 from typing import (Any, Callable, Dict, Generator, Iterator, List,
                     Mapping, MutableMapping, MutableSequence, Optional, Set, Tuple,
@@ -53,6 +53,10 @@
 if TYPE_CHECKING:
     from .provenance import ProvenanceProfile  # pylint: disable=unused-import
 
+if PY3:
+    from collections.abc import Iterable # only works on python 3.3+
+else:
+    from collections import Iterable  # pylint: disable=unused-import
 
 class LogAsDebugFilter(logging.Filter):
     def __init__(self, name, parent):  # type: (Text, logging.Logger) -> None
@@ -722,7 +726,14 @@ def inc(d):  # type: (List[int]) -> None
                 if isinstance(arg, MutableMapping):
                     arg = copy.deepcopy(arg)
                     if arg.get("position"):
-                        arg["position"] = [arg["position"], i]
+                        position = arg.get("position")
+                        if isinstance(position, str):  # no need to test the
+                                                       # CWLVersion as the v1.0
+                                                       # schema only allows ints
+                            position = builder.do_eval(position)
+                            if position is None:
+                                position = 0
+                        arg["position"] = [position, i]
                     else:
                         arg["position"] = [0, i]
                     bindings.append(arg)

cwltool/schemas/v1.1.0-dev1/CommandLineTool.yml

@@ -1,3 +1,4 @@
+saladVersion: v1.1
 $base: "https://w3id.org/cwl/cwl#"
 
 $namespaces:
@@ -56,8 +57,8 @@ $graph:
         * When defining record types with `CommandInputRecordSchema`, fields of
           type `File` may now include `format`, `loadContents`,
           `secondaryFiles` and `streamable`.
-        * `CommandInputRecordSchema`, `CommandOutputRecordSchema`,
-          `CommandInputEnumSchema and `CommandInputArraySchema` now have an optional
+        * `CommandInputRecordSchema`, `CommandOutputRecordSchema`, 
+          `CommandInputEnumSchema and `CommandInputArraySchema` now have an optional 
           `doc` field.
         * `inputBinding` has been added as an optional field for
           `CommandInputRecordSchema` (was previously in CWL `draft-3` but
@@ -88,8 +89,8 @@ $graph:
         * Added `InputParameter.loadContents` field. Use of `loadContents` in
           `InputBinding` is deprecated; it is preserved for v1.0 backwards
            compatability and will be removed in CWL v2.0.
-        * [Added](#ToolTimeLimit) `ToolTimeLimit` feature, allowing to set an upper limit on the
-          execution time of a CommandLineTool.
+        * [Added](#ToolTimeLimit) `ToolTimeLimit` feature, allows setting
+          an upper limit on the execution time of a CommandLineTool.
         * [Added](#WorkReuse) `WorkReuse` feature, allowing to enable or disable the reuse
           behavior for a particular tool or step for implementations that
           support reusing output from past work.
@@ -100,6 +101,15 @@ $graph:
         * [Added](#LoadListingRequirement) `LoadListingRequirement`
           and [loadListing](#LoadContents) to control whether and how
           `Directory` listings should be loaded for use in expressions.
+        * The position field of the [CommandLineBinding](#CommandLineBinding) can
+          now be calculated from a CWL Expression.
+        * The exit code of a CommandLineTool invocation is now
+          available to expressions in `outputEval` as `runtime.exitCode`
+        * [Better explain](#map) the `map<…>` notation that has existed since v1.0.
+        * Fixed schema error where the `type` field inside the `inputs` and
+          `outputs` field was incorrectly listed as optional.
+        * For multi-Process CWL documents, if no particular process is named then
+          the process with the `id` of `#main` is chosen.
 
       See also the [CWL Workflow Description, v1.1.0-dev1 changelog](Workflow.html#Changelog).
 
@@ -197,8 +207,14 @@ $graph:
 
   fields:
     - name: position
-      type: int?
-      doc: "The sorting key.  Default position is 0."
+      type: [ int, Expression, string, "null" ]
+      doc: |
+        The sorting key.  Default position is 0. If the inputBinding is
+        associated with an input parameter, then the value of `self` in the
+        expression will be the value of the input parameter.  Input parameter
+        defaults (as specified by the `InputParameter.default` field) must be
+        applied before evaluating the expression. Expressions must return a
+        single value of type int or a null.
     - name: prefix
       type: string?
       doc: "Command line prefix to add before the value."
@@ -316,13 +332,15 @@ $graph:
         - string
         - Expression
       doc: |
-        Evaluate an expression to generate the output value.  If `glob` was
-        specified, the value of `self` must be an array containing file objects
-        that were matched.  If no files were matched, `self` must be a zero
-        length array; if a single file was matched, the value of `self` is an
-        array of a single element.  Additionally, if `loadContents` is `true`,
-        the File objects must include up to the first 64 KiB of file contents
-        in the `contents` field.
+        Evaluate an expression to generate the output value.  If
+        `glob` was specified, the value of `self` must be an array
+        containing file objects that were matched.  If no files were
+        matched, `self` must be a zero length array; if a single file
+        was matched, the value of `self` is an array of a single
+        element.  Additionally, if `loadContents` is `true`, the File
+        objects must include up to the first 64 KiB of file contents
+        in the `contents` field.  The exit code of the process is
+        available in the expression as `runtime.exitCode`.
 
 - name: CommandLineBindable
   type: record
@@ -436,7 +454,6 @@ $graph:
   fields:
     - name: type
       type:
-        - "null"
         - CWLType
         - stdin
         - CommandInputRecordSchema
@@ -471,7 +488,6 @@ $graph:
   fields:
     - name: type
       type:
-        - "null"
         - CWLType
         - stdout
         - stderr
@@ -1097,36 +1113,6 @@ $graph:
       doc: Maximum reserved filesystem based storage for the designated output directory, in mebibytes (2**20)
 
 
-- type: record
-  name: ToolTimeLimit
-  extends: ProcessRequirement
-  doc: |
-    Set an upper limit on the execution time of a CommandLineTool.
-    A tool execution which exceeds the time limit may
-    be preemptively terminated and considered failed.  May also be
-    used by batch systems to make scheduling decisions.
-
-    Limit applies only to the command execution time. Time for
-    additional steps, such as file download/upload, docker pull or
-    expression evaluation is not considered for exceeding time limit.
-
-    If ToolTimeLimit is set on a workflow level, it propagates down to
-    individual processes, in line with requirement inheritance rules.
-    ToolTimeLimit is not used to limit the execution time of a Workflow.
-  fields:
-    - name: class
-      type: string
-      doc: "Always 'ToolTimeLimit'"
-      jsonldPredicate:
-        "_id": "@type"
-        "_type": "@vocab"
-    - name: timelimit
-      type: [long, string, Expression]
-      doc: |
-        The time limit, in seconds.  A time limit of zero means no
-        time limit.  Negative time limits are an error.
-
-
 - type: record
   name: WorkReuse
   extends: ProcessRequirement
@@ -1226,3 +1212,27 @@ $graph:
         "_type": "@vocab"
     inplaceUpdate:
       type: boolean
+
+- type: record
+  name: ToolTimeLimit
+  extends: ProcessRequirement
+  doc: |
+    Set an upper limit on the execution time of a CommandLineTool.
+    A CommandLineTool whose execution duration exceeds the time
+    limit may be preemptively terminated and considered failed.
+    May also be used by batch systems to make scheduling decisions.
+    The execution duration excludes external operations, such as
+    staging of files, pulling a docker image etc, and only counts
+    wall-time for the execution of the command line itself.
+  fields:
+    - name: class
+      type: string
+      doc: "Always 'ToolTimeLimit'"
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+    - name: timelimit
+      type: [long, string, Expression]
+      doc: |
+        The time limit, in seconds.  A time limit of zero means no
+        time limit.  Negative time limits are an error.

cwltool/schemas/v1.1.0-dev1/CommonWorkflowLanguage.yml

@@ -1,3 +1,4 @@
+saladVersion: v1.1
 $base: "https://w3id.org/cwl/cwl#"
 
 $namespaces:

cwltool/schemas/v1.1.0-dev1/Process.yml

@@ -1,3 +1,4 @@
+saladVersion: v1.1
 $base: "https://w3id.org/cwl/cwl#"
 
 $namespaces:

cwltool/schemas/v1.1.0-dev1/README.md

@@ -1,71 +1,9 @@
-# Common Workflow Language Specifications, v1.1.0-dev1
+Hello!
 
-<!-- remove this before v1.1 is final! -->
-WARNING, this is a draft and in progress! Development of CWL v1.1 is coordinated at https://github.com/common-workflow-language/common-workflow-language/milestone/6
+This repo represent the current working proposal for v1.1 of the Common Workflow Language standard.
 
-The CWL specifications are divided up into several documents.
+Proposed changes that aren't yet implemented are tracked at https://github.com/common-workflow-language/common-workflow-language/milestone/6
 
-The [User Guide](http://www.commonwl.org/user_guide/) provides a gentle
-introduction to writing CWL command line tools and workflows.
+You can render this using https://github.com/common-workflow-language/cwl-website/blob/master/website.sh
 
-The [Command Line Tool Description Specification](CommandLineTool.html)
-specifies the document schema and execution semantics for wrapping and
-executing command line tools.
-
-The [Workflow Description Specification](Workflow.html) specifies the document
-schema and execution semantics for composing workflows from components such as
-command line tools and other workflows.
-
-The
-[Semantic Annotations for Linked Avro Data (SALAD) Specification](SchemaSalad.html)
-specifies the preprocessing steps that must be applied when loading CWL
-documents and the schema language used to write the above specifications.
-
-Also available are inheritance graphs (as SVG images) for the [Schema Salad object model](salad.svg) and the [CWL object model](cwl.svg).
-
-# Running the CWL conformance tests
-
-1. Install a CWL runner of your choice. The reference runner can be installed as
-the default runner by doing:
-```
-pip install cwlref-runner
-```
-
-2. Install the CWL test parser:
-
-```
-pip install cwltest
-```
-You may need to activate a virtualenv first, or do a local install by adding `--user` after `install` above.
-
-3. From within a copy of [this repository](https://github.com/common-workflow-language/cwl-v1.1) (e.g. cwl-v1.1) execute the main test script
-```
-./run_test.sh
-```
-
-If the CWL runner isn't installed as `cwl-runner` then you can specify the name:
-
-```
-./run_test.sh RUNNER=cwltool
-```
-
-You can also specify additional options that are specific for the particular CWL runner you are using.
-For example, with CWL reference runner you can turn on parallel execution mode:
-
-```
-./run_test.sh RUNNER=cwltool EXTRA=--parallel
-```
-
-This can be combined with launching more than one CWL conformance test at once with `-j`:
-
-```
-./run_test.sh -j4 RUNNER=cwltool EXTRA=--parallel
-```
-
-
-For details of options you can pass to the test script, do:
-```
-./run_test.sh --help
-```
-
-The full test suite takes about 10 minutes to run
+The current rendering is at https://www.commonwl.org/v1.1.0-dev1/