Skip to content

Commit c5ce229

Browse files
lkk7lkk7
authored
Merge branch 'main' into fix-tests-pydot
2 parents fcab10e + 73b742f commit c5ce229

File tree

130 files changed

+12872
-374
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

130 files changed

+12872
-374
lines changed

‎.github/workflows/ci-tests.yml

+2
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,8 @@ jobs:
173173
- cwl-version: v1.2
174174
container: docker
175175
extras: "--fast-parser"
176+
- cwl-version: v1.3.0-dev1
177+
extras: "--relax-path-checks"
176178

177179
steps:
178180
- uses: actions/checkout@v4

‎MANIFEST.in

+5
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ recursive-include mypy-stubs *.pyi *.py
77
include tests/*
88
include tests/cwl-conformance/cwltool-conftest.py
99
include tests/loop/*
10+
include tests/loop-ext/*
1011
include tests/tmp1/tmp2/tmp3/.gitkeep
1112
include tests/tmp4/alpha/*
1213
include tests/wf/*
@@ -54,6 +55,10 @@ include cwltool/schemas/v1.2/*.yml
5455
include cwltool/schemas/v1.2/*.md
5556
include cwltool/schemas/v1.2/salad/schema_salad/metaschema/*.yml
5657
include cwltool/schemas/v1.2/salad/schema_salad/metaschema/*.md
58+
include cwltool/schemas/v1.3.0-dev1/*.yml
59+
include cwltool/schemas/v1.3.0-dev1/*.md
60+
include cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/*.yml
61+
include cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/*.md
5762
include cwltool/extensions.yml
5863
include cwltool/extensions-v1.1.yml
5964
include cwltool/extensions-v1.2.yml

‎Makefile

+3-3
Original file line numberDiff line numberDiff line change
@@ -160,11 +160,11 @@ diff-cover.html: coverage.xml
160160

161161
## test : run the cwltool test suite
162162
test: $(PYSOURCES)
163-
python3 -m pytest -rs ${PYTEST_EXTRA}
163+
python3 -m pytest -rsfE ${PYTEST_EXTRA}
164164

165165
## testcov : run the cwltool test suite and collect coverage
166166
testcov: $(PYSOURCES)
167-
python3 -m pytest -rs --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}
167+
python3 -m pytest -rsfE --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}
168168

169169
sloccount.sc: $(PYSOURCES) Makefile
170170
sloccount --duplicates --wide --details $^ > $@
@@ -183,7 +183,7 @@ mypy: $(PYSOURCES)
183183

184184
mypyc: $(PYSOURCES)
185185
MYPYPATH=mypy-stubs CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \
186-
&& pytest -rs -vv ${PYTEST_EXTRA}
186+
&& pytest -rsfE -vv ${PYTEST_EXTRA}
187187

188188
shellcheck: FORCE
189189
shellcheck build-cwltool-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \

‎README.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#############################################################################################
2-
``cwltool``: The reference reference implementation of the Common Workflow Language standards
2+
``cwltool``: The reference implementation of the Common Workflow Language standards
33
#############################################################################################
44

55
|Linux Status| |Coverage Status| |Docs Status|

‎conformance-test.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ venv() {
1616
# VERSION=v1.2 GIT_TARGET=main CONTAINER=podman ./conformance_test.sh
1717

1818
# Version of the standard to test against
19-
# Current options: v1.0, v1.1, v1.2
19+
# Current options: v1.0, v1.1, v1.2, v1.3.0-dev1
2020
VERSION=${VERSION:-"v1.2"}
2121

2222
# Which commit of the standard's repo to use
@@ -121,7 +121,7 @@ if (( "${#exclusions[*]}" > 0 )); then
121121
fi
122122

123123
# Build command
124-
TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n logical --dist worksteal -rs --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
124+
TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n logical --dist worksteal -rsfE --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
125125
if [[ -n "${EXCLUDE}" ]] ; then
126126
TEST_COMMAND="${TEST_COMMAND} --cwl-exclude ${EXCLUDE}"
127127
fi

‎cwltool/checker.py

+9-18
Original file line numberDiff line numberDiff line change
@@ -517,42 +517,33 @@ def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -
517517

518518

519519
def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool:
520-
"""Check if a step contains a http://commonwl.org/cwltool#Loop requirement with `all` outputMethod."""
520+
"""Check if a step contains a `loop` directive with `all` outputMethod."""
521521
source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id)
522522
if source_step is not None:
523-
for requirement in source_step.get("requirements", []):
524-
if (
525-
requirement["class"] == "http://commonwl.org/cwltool#Loop"
526-
and requirement.get("outputMethod") == "all"
527-
):
528-
return True
523+
if source_step.get("loop") is not None and source_step.get("outputMethod") == "all":
524+
return True
529525
return False
530526

531527

532528
def loop_checker(steps: Iterator[MutableMapping[str, Any]]) -> None:
533529
"""
534-
Check http://commonwl.org/cwltool#Loop requirement compatibility with other directives.
530+
Check `loop` compatibility with other directives.
535531
536-
:raises ValidationException: If there is an incompatible combination between
537-
cwltool:loop and 'scatter' or 'when'.
532+
:raises ValidationException: If there is an incompatible combination between `loop` and `scatter`.
538533
"""
539534
exceptions = []
540535
for step in steps:
541-
requirements = {
542-
**{h["class"]: h for h in step.get("hints", [])},
543-
**{r["class"]: r for r in step.get("requirements", [])},
544-
}
545-
if "http://commonwl.org/cwltool#Loop" in requirements:
546-
if "when" in step:
536+
if "loop" in step:
537+
if "when" not in step:
547538
exceptions.append(
548539
SourceLine(step, "id").makeError(
549-
"The `cwltool:Loop` clause is not compatible with the `when` directive."
540+
"The `when` clause is mandatory when the `loop` directive is defined."
550541
)
551542
)
552543
if "scatter" in step:
553544
exceptions.append(
554545
SourceLine(step, "id").makeError(
555-
"The `cwltool:Loop` clause is not compatible with the `scatter` directive."
546+
"The `loop` clause is not compatible with the `scatter` directive."
556547
)
557548
)
558549
if exceptions:

‎cwltool/command_line_tool.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -806,7 +806,7 @@ def remove_dirname(d: CWLObjectType) -> None:
806806
def job(
807807
self,
808808
job_order: CWLObjectType,
809-
output_callbacks: Optional[OutputCallbackType],
809+
output_callbacks: OutputCallbackType,
810810
runtimeContext: RuntimeContext,
811811
) -> Generator[Union[JobBase, CallbackJob], None, None]:
812812
workReuse, _ = self.get_requirement("WorkReuse")

‎cwltool/errors.py

+14-6
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,17 @@
1-
class WorkflowException(Exception):
2-
pass
1+
"""Common errors.
2+
3+
WorkflowException and GraphTargetMissingException are aliased to
4+
equivalent errors from cwl_utils.errors and re-exported by this module
5+
to avoid breaking the interface for other code.
6+
7+
"""
8+
9+
# flake8: noqa: F401
10+
11+
from cwl_utils.errors import WorkflowException as WorkflowException
12+
13+
14+
from cwl_utils.errors import GraphTargetMissingException as GraphTargetMissingException
315

416

517
class UnsupportedRequirement(WorkflowException):
@@ -8,7 +20,3 @@ class UnsupportedRequirement(WorkflowException):
820

921
class ArgumentException(Exception):
1022
"""Mismatched command line arguments provided."""
11-
12-
13-
class GraphTargetMissingException(WorkflowException):
14-
"""When a ``$graph`` is encountered and there is no target and no ``main``/``#main``."""

‎cwltool/extensions-v1.3.yml

+121
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
$base: http://commonwl.org/cwltool#
2+
$namespaces:
3+
cwl: "https://w3id.org/cwl/cwl#"
4+
$graph:
5+
- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
6+
7+
- name: Secrets
8+
type: record
9+
inVocab: false
10+
extends: cwl:ProcessRequirement
11+
fields:
12+
class:
13+
type: string
14+
doc: "Always 'Secrets'"
15+
jsonldPredicate:
16+
"_id": "@type"
17+
"_type": "@vocab"
18+
secrets:
19+
type: string[]
20+
doc: |
21+
List one or more input parameters that are sensitive (such as passwords)
22+
which will be deliberately obscured from logging.
23+
jsonldPredicate:
24+
"_type": "@id"
25+
refScope: 0
26+
27+
28+
- name: ProcessGenerator
29+
type: record
30+
inVocab: true
31+
extends: cwl:Process
32+
documentRoot: true
33+
fields:
34+
- name: class
35+
jsonldPredicate:
36+
"_id": "@type"
37+
"_type": "@vocab"
38+
type: string
39+
- name: run
40+
type: [string, cwl:Process]
41+
jsonldPredicate:
42+
_id: "cwl:run"
43+
_type: "@id"
44+
subscope: run
45+
doc: |
46+
Specifies the process to run.
47+
48+
- name: MPIRequirement
49+
type: record
50+
inVocab: false
51+
extends: cwl:ProcessRequirement
52+
doc: |
53+
Indicates that a process requires an MPI runtime.
54+
fields:
55+
- name: class
56+
type: string
57+
doc: "Always 'MPIRequirement'"
58+
jsonldPredicate:
59+
"_id": "@type"
60+
"_type": "@vocab"
61+
- name: processes
62+
type: [int, cwl:Expression]
63+
doc: |
64+
The number of MPI processes to start. If you give a string,
65+
this will be evaluated as a CWL Expression and it must
66+
evaluate to an integer.
67+
68+
- name: CUDARequirement
69+
type: record
70+
extends: cwl:ProcessRequirement
71+
inVocab: false
72+
doc: |
73+
Require support for NVIDA CUDA (GPU hardware acceleration).
74+
fields:
75+
class:
76+
type: string
77+
doc: 'cwltool:CUDARequirement'
78+
jsonldPredicate:
79+
_id: "@type"
80+
_type: "@vocab"
81+
cudaVersionMin:
82+
type: string
83+
doc: |
84+
Minimum CUDA version to run the software, in X.Y format. This
85+
corresponds to a CUDA SDK release. When running directly on
86+
the host (not in a container) the host must have a compatible
87+
CUDA SDK (matching the exact version, or, starting with CUDA
88+
11.3, matching major version). When run in a container, the
89+
container image should provide the CUDA runtime, and the host
90+
driver is injected into the container. In this case, because
91+
CUDA drivers are backwards compatible, it is possible to
92+
use an older SDK with a newer driver across major versions.
93+
94+
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
95+
details.
96+
cudaComputeCapability:
97+
type:
98+
- 'string'
99+
- 'string[]'
100+
doc: |
101+
CUDA hardware capability required to run the software, in X.Y
102+
format.
103+
104+
* If this is a single value, it defines only the minimum
105+
compute capability. GPUs with higher capability are also
106+
accepted.
107+
108+
* If it is an array value, then only select GPUs with compute
109+
capabilities that explicitly appear in the array.
110+
cudaDeviceCountMin:
111+
type: ['null', int, cwl:Expression]
112+
default: 1
113+
doc: |
114+
Minimum number of GPU devices to request. If not specified,
115+
same as `cudaDeviceCountMax`. If neither are specified,
116+
default 1.
117+
cudaDeviceCountMax:
118+
type: ['null', int, cwl:Expression]
119+
doc: |
120+
Maximum number of GPU devices to request. If not specified,
121+
same as `cudaDeviceCountMin`.

‎cwltool/process.py

+2-5
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ def filter(self, record: logging.LogRecord) -> bool:
125125
]
126126

127127
cwl_files = (
128+
"Base.yml",
128129
"Workflow.yml",
129130
"CommandLineTool.yml",
130131
"CommonWorkflowLanguage.yml",
@@ -1046,10 +1047,6 @@ def validate_hints(self, avsc_names: Names, hints: List[CWLObjectType], strict:
10461047
sl = SourceLine(hints, i, ValidationException, debug)
10471048
with sl:
10481049
classname = cast(str, r["class"])
1049-
if classname == "http://commonwl.org/cwltool#Loop":
1050-
raise ValidationException(
1051-
"http://commonwl.org/cwltool#Loop is valid only under requirements."
1052-
)
10531050
avroname = classname
10541051
if classname in self.doc_loader.vocab:
10551052
avroname = avro_type_name(self.doc_loader.vocab[classname])
@@ -1078,7 +1075,7 @@ def visit(self, op: Callable[[CommentedMap], None]) -> None:
10781075
def job(
10791076
self,
10801077
job_order: CWLObjectType,
1081-
output_callbacks: Optional[OutputCallbackType],
1078+
output_callbacks: OutputCallbackType,
10821079
runtimeContext: RuntimeContext,
10831080
) -> JobsGeneratorType:
10841081
pass

‎cwltool/procgenerator.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ def receive_output(self, jobout: Optional[CWLObjectType], processStatus: str) ->
3030
def job(
3131
self,
3232
job_order: CWLObjectType,
33-
output_callbacks: Optional[OutputCallbackType],
33+
output_callbacks: OutputCallbackType,
3434
runtimeContext: RuntimeContext,
3535
) -> JobsGeneratorType:
3636
try:
@@ -41,7 +41,7 @@ def job(
4141
while self.processStatus is None:
4242
yield None
4343

44-
if self.processStatus != "success" and output_callbacks:
44+
if self.processStatus != "success":
4545
output_callbacks(self.jobout, self.processStatus)
4646
return
4747

@@ -89,7 +89,7 @@ def __init__(
8989
def job(
9090
self,
9191
job_order: CWLObjectType,
92-
output_callbacks: Optional[OutputCallbackType],
92+
output_callbacks: OutputCallbackType,
9393
runtimeContext: RuntimeContext,
9494
) -> JobsGeneratorType:
9595
return ProcessGeneratorJob(self).job(job_order, output_callbacks, runtimeContext)

0 commit comments

Comments
 (0)