Skip to content

Commit db82bbb

Browse files
mr-cmr-c
committed
fix empty scatter bug
1 parent 8763bed commit db82bbb

9 files changed

+327
-6
lines changed

‎cwltool/workflow_job.py‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ def nested_crossproduct_scatter(
172172
runtimeContext: RuntimeContext,
173173
) -> JobsGeneratorType:
174174
scatter_key = scatter_keys[0]
175-
jobl = len(cast(Sized, joborder[scatter_key]))
175+
jobl = len(cast(Sized, joborder[scatter_key])) if joborder[scatter_key] else 0
176176
output = {} # type: ScatterDestinationsType
177177
for i in process.tool["outputs"]:
178178
output[i["id"]] = [None] * jobl
@@ -219,7 +219,9 @@ def crossproduct_size(
219219
ssum = len(cast(Sized, joborder[scatter_key]))
220220
else:
221221
ssum = 0
222-
for _ in range(0, len(cast(Sized, joborder[scatter_key]))):
222+
for _ in range(
223+
0, len(cast(Sized, joborder[scatter_key])) if joborder[scatter_key] else 0
224+
):
223225
ssum += crossproduct_size(joborder, scatter_keys[1:])
224226
return ssum
225227

@@ -252,7 +254,7 @@ def _flat_crossproduct_scatter(
252254
) -> Tuple[List[Optional[JobsGeneratorType]], int,]:
253255
"""Inner loop."""
254256
scatter_key = scatter_keys[0]
255-
jobl = len(cast(Sized, joborder[scatter_key]))
257+
jobl = len(cast(Sized, joborder[scatter_key])) if joborder[scatter_key] else 0
256258
steps = [] # type: List[Optional[JobsGeneratorType]]
257259
put = startindex
258260
for index in range(0, jobl):
@@ -292,8 +294,8 @@ def dotproduct_scatter(
292294
jobl = None # type: Optional[int]
293295
for key in scatter_keys:
294296
if jobl is None:
295-
jobl = len(cast(Sized, joborder[key]))
296-
elif jobl != len(cast(Sized, joborder[key])):
297+
jobl = len(cast(Sized, joborder[key])) if joborder[key] else 0
298+
elif jobl != len(cast(Sized, joborder[key])) if joborder[key] else 0:
297299
raise WorkflowException(
298300
"Length of input arrays must be equal when performing "
299301
"dotproduct scatter."
@@ -729,7 +731,9 @@ def valueFromFunc(
729731
runtimeContext.postScatterEval = postScatterEval
730732

731733
emptyscatter = [
732-
shortname(s) for s in scatter if len(cast(Sized, inputobj[s])) == 0
734+
shortname(s)
735+
for s in scatter
736+
if not inputobj[s] or len(cast(Sized, inputobj[s])) == 0
733737
]
734738
if emptyscatter:
735739
_logger.warning(

‎tests/test_conditionals.py‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,158 @@ def test_conditional_step_no_inputs() -> None:
1717
stderr = re.sub(r"\s\s+", " ", stderr)
1818
assert err_code == 0, stderr
1919
assert result is None
20+
21+
22+
def test_conditional_scatter_missing_input() -> None:
23+
"""Test that scattering a missing array skips execution."""
24+
err_code, stdout, stderr = get_main_output(
25+
[
26+
get_data("tests/wf/scatter_before_when.cwl"),
27+
]
28+
)
29+
result = json.loads(stdout)["optional_echoed_messages"]
30+
stderr = re.sub(r"\s\s+", " ", stderr)
31+
assert err_code == 0, stderr
32+
assert result == []
33+
34+
35+
def test_scatter_empty_array() -> None:
36+
"""Test that scattering an empty array skips execution."""
37+
err_code, stdout, stderr = get_main_output(
38+
[
39+
get_data("tests/wf/scatter_before_when.cwl"),
40+
get_data("tests/wf/scatter_before_when-inp_empty.yaml"),
41+
]
42+
)
43+
result = json.loads(stdout)["optional_echoed_messages"]
44+
stderr = re.sub(r"\s\s+", " ", stderr)
45+
assert err_code == 0, stderr
46+
assert result == []
47+
48+
49+
def test_scatter_and_conditional() -> None:
50+
"""Test scattering a partially empty array with a conditional."""
51+
err_code, stdout, stderr = get_main_output(
52+
[
53+
get_data("tests/wf/scatter_before_when.cwl"),
54+
get_data("tests/wf/scatter_before_when-inp.yaml"),
55+
]
56+
)
57+
result = json.loads(stdout)["optional_echoed_messages"]
58+
stderr = re.sub(r"\s\s+", " ", stderr)
59+
assert err_code == 0, stderr
60+
assert result == ["We\n", "come\n", "in\n", "peace\n"]
61+
62+
63+
def test_scatter_dotproduct_empty_arrays() -> None:
64+
"""Test that dotproduct scattering empty arrays skips execution."""
65+
err_code, stdout, stderr = get_main_output(
66+
[
67+
get_data("tests/wf/scatter_before_when_dotproduct.cwl"),
68+
]
69+
)
70+
result = json.loads(stdout)["optional_echoed_messages"]
71+
stderr = re.sub(r"\s\s+", " ", stderr)
72+
assert err_code == 0, stderr
73+
assert result == []
74+
75+
76+
def test_scatter_dotproduct_and_conditional() -> None:
77+
"""Test dotproduct scattering with partially empty arrays."""
78+
err_code, stdout, stderr = get_main_output(
79+
[
80+
get_data("tests/wf/scatter_before_when_dotproduct.cwl"),
81+
get_data("tests/wf/scatter_before_when_dotproduct-inp.yaml"),
82+
]
83+
)
84+
result = json.loads(stdout)["optional_echoed_messages"]
85+
stderr = re.sub(r"\s\s+", " ", stderr)
86+
assert err_code == 0, stderr
87+
assert result == [
88+
"We Never\n",
89+
"Come Out\n",
90+
"In Anything But\n",
91+
"Peace -- The Aliens\n",
92+
]
93+
94+
95+
def test_scatter_nested_crossproduct_empty_arrays() -> None:
96+
"""Test that nested_dotproduct scattering empty arrays skips execution."""
97+
err_code, stdout, stderr = get_main_output(
98+
[
99+
get_data("tests/wf/scatter_before_when-nested_crossproduct.cwl"),
100+
]
101+
)
102+
result = json.loads(stdout)["optional_echoed_messages"]
103+
stderr = re.sub(r"\s\s+", " ", stderr)
104+
assert err_code == 0, stderr
105+
assert result == []
106+
107+
108+
def test_scatter_nested_crossproduct_and_conditional() -> None:
109+
"""Test nested_crossproduct scattering with partially empty arrays."""
110+
err_code, stdout, stderr = get_main_output(
111+
[
112+
get_data("tests/wf/scatter_before_when-nested_crossproduct.cwl"),
113+
get_data("tests/wf/scatter_before_when_dotproduct-inp.yaml"),
114+
]
115+
)
116+
result = json.loads(stdout)["optional_echoed_messages"]
117+
stderr = re.sub(r"\s\s+", " ", stderr)
118+
assert err_code == 0, stderr
119+
assert result == [
120+
["We Never\n", "We Out\n", "We Anything But\n", None, "We -- The Aliens\n"],
121+
[
122+
"Come Never\n",
123+
"Come Out\n",
124+
"Come Anything But\n",
125+
None,
126+
"Come -- The Aliens\n",
127+
],
128+
["In Never\n", "In Out\n", "In Anything But\n", None, "In -- The Aliens\n"],
129+
[None, None, None, None, None],
130+
]
131+
132+
133+
def test_scatter_flat_crossproduct_empty_arrays() -> None:
134+
"""Test that flat_dotproduct scattering empty arrays skips execution."""
135+
err_code, stdout, stderr = get_main_output(
136+
[
137+
get_data("tests/wf/scatter_before_when-flat_crossproduct.cwl"),
138+
]
139+
)
140+
result = json.loads(stdout)["optional_echoed_messages"]
141+
stderr = re.sub(r"\s\s+", " ", stderr)
142+
assert err_code == 0, stderr
143+
assert result == []
144+
145+
146+
def test_scatter_flat_crossproduct_and_conditional() -> None:
147+
"""Test flat_crossproduct scattering with partially empty arrays."""
148+
err_code, stdout, stderr = get_main_output(
149+
[
150+
get_data("tests/wf/scatter_before_when-flat_crossproduct.cwl"),
151+
get_data("tests/wf/scatter_before_when_dotproduct-inp.yaml"),
152+
]
153+
)
154+
result = json.loads(stdout)["optional_echoed_messages"]
155+
stderr = re.sub(r"\s\s+", " ", stderr)
156+
assert err_code == 0, stderr
157+
assert result == [
158+
"We Never\n",
159+
"We Out\n",
160+
"We Anything But\n",
161+
"We -- The Aliens\n",
162+
"Come Never\n",
163+
"Come Out\n",
164+
"Come Anything But\n",
165+
"Come -- The Aliens\n",
166+
"In Never\n",
167+
"In Out\n",
168+
"In Anything But\n",
169+
"In -- The Aliens\n",
170+
"Peace Never\n",
171+
"Peace Out\n",
172+
"Peace Anything But\n",
173+
"Peace -- The Aliens\n",
174+
]

‎tests/wf/scatter_before_when-flat_crossproduct.cwl‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
cwlVersion: v1.2
2+
class: Workflow
3+
4+
requirements:
5+
ScatterFeatureRequirement: {}
6+
InlineJavascriptRequirement: {}
7+
StepInputExpressionRequirement: {}
8+
9+
inputs:
10+
messages:
11+
type:
12+
- "null"
13+
- type: array
14+
items: [string, "null"]
15+
extras:
16+
type:
17+
- "null"
18+
- type: array
19+
items: [string, "null"]
20+
21+
steps:
22+
optional_echo_scatter:
23+
when: $(inputs.messages !== null && inputs.extra !== null)
24+
run: ../echo.cwl
25+
scatter: [messages, extra]
26+
scatterMethod: flat_crossproduct
27+
in:
28+
extra: extras
29+
messages: messages
30+
inp:
31+
valueFrom: $(inputs.messages) $(inputs.extra)
32+
out: [out]
33+
34+
outputs:
35+
optional_echoed_messages:
36+
type: string[]?
37+
pickValue: all_non_null
38+
outputSource: optional_echo_scatter/out

‎tests/wf/scatter_before_when-inp.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
messages:
2+
- We
3+
- come
4+
- in
5+
- null
6+
- peace

‎tests/wf/scatter_before_when-inp_empty.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
messages: []
2+

‎tests/wf/scatter_before_when-nested_crossproduct.cwl‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
cwlVersion: v1.2
2+
class: Workflow
3+
4+
requirements:
5+
ScatterFeatureRequirement: {}
6+
InlineJavascriptRequirement: {}
7+
StepInputExpressionRequirement: {}
8+
9+
inputs:
10+
messages:
11+
type:
12+
- "null"
13+
- type: array
14+
items: [string, "null"]
15+
extras:
16+
type:
17+
- "null"
18+
- type: array
19+
items: [string, "null"]
20+
21+
steps:
22+
optional_echo_scatter:
23+
when: $(inputs.messages !== null && inputs.extra !== null)
24+
run: ../echo.cwl
25+
scatter: [messages, extra]
26+
scatterMethod: nested_crossproduct
27+
in:
28+
extra: extras
29+
messages: messages
30+
inp:
31+
valueFrom: $(inputs.messages) $(inputs.extra)
32+
out: [out]
33+
34+
outputs:
35+
optional_echoed_messages:
36+
type: string[]?
37+
pickValue: all_non_null
38+
outputSource: optional_echo_scatter/out

‎tests/wf/scatter_before_when.cwl‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
cwlVersion: v1.2
2+
class: Workflow
3+
4+
requirements:
5+
ScatterFeatureRequirement: {}
6+
InlineJavascriptRequirement: {}
7+
8+
inputs:
9+
messages:
10+
type:
11+
- "null"
12+
- type: array
13+
items: [string, "null"]
14+
15+
steps:
16+
optional_echo_scatter:
17+
when: $(inputs.inp !== null)
18+
run: ../echo.cwl
19+
scatter: inp
20+
in:
21+
inp: messages
22+
out: [out]
23+
24+
outputs:
25+
optional_echoed_messages:
26+
type: string[]?
27+
pickValue: all_non_null
28+
outputSource: optional_echo_scatter/out

‎tests/wf/scatter_before_when_dotproduct-inp.yaml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
messages:
2+
- We
3+
- Come
4+
- In
5+
- null
6+
- Peace
7+
extras:
8+
- Never
9+
- Out
10+
- Anything But
11+
- null
12+
- "-- The Aliens"

‎tests/wf/scatter_before_when_dotproduct.cwl‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
cwlVersion: v1.2
2+
class: Workflow
3+
4+
requirements:
5+
ScatterFeatureRequirement: {}
6+
InlineJavascriptRequirement: {}
7+
StepInputExpressionRequirement: {}
8+
9+
inputs:
10+
messages:
11+
type:
12+
- "null"
13+
- type: array
14+
items: [string, "null"]
15+
extras:
16+
type:
17+
- "null"
18+
- type: array
19+
items: [string, "null"]
20+
21+
steps:
22+
optional_echo_scatter:
23+
when: $(inputs.messages !== null && inputs.extras !== null)
24+
run: ../echo.cwl
25+
scatter: [messages, extra]
26+
scatterMethod: dotproduct
27+
in:
28+
extra: extras
29+
messages: messages
30+
inp:
31+
valueFrom: $(inputs.messages) $(inputs.extra)
32+
out: [out]
33+
34+
outputs:
35+
optional_echoed_messages:
36+
type: string[]?
37+
pickValue: all_non_null
38+
outputSource: optional_echo_scatter/out

0 commit comments

Comments
 (0)