Skip to content

Commit 0f3c040

Browse files
mr-cmr-c
committed
check the initial working directory before the cache attempt
1 parent d439cd3 commit 0f3c040

8 files changed

+10255
-136
lines changed

‎MANIFEST.in

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ include tests/loop-ext/*
1111
include tests/tmp1/tmp2/tmp3/.gitkeep
1212
include tests/tmp4/alpha/*
1313
include tests/wf/*
14+
include tests/wf/inp-filelist.txt
1415
include tests/wf/operation/*
1516
include tests/override/*
1617
include tests/reloc/*.cwl

‎cwltool/command_line_tool.py

+12-1
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,14 @@ def updatePathmap(self, outdir: str, pathmap: PathMapper, fn: CWLObjectType) ->
492492
for ls in cast(list[CWLObjectType], fn.get("listing", [])):
493493
self.updatePathmap(os.path.join(outdir, cast(str, fn["basename"])), pathmap, ls)
494494

495-
def _initialworkdir(self, j: JobBase, builder: Builder) -> None:
495+
def _initialworkdir(self, j: Optional[JobBase], builder: Builder) -> None:
496+
"""
497+
Test and initialize the working directory.
498+
499+
:param j: A :py:class:`~cwltool.job.CommandLineJob` or a
500+
specialized container-based job.
501+
If 'None', then only tests will be performed, no setup.
502+
"""
496503
initialWorkdir, _ = self.get_requirement("InitialWorkDirRequirement")
497504
if initialWorkdir is None:
498505
return
@@ -760,6 +767,9 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None:
760767
"is in 'requirements'."
761768
)
762769

770+
if j is None:
771+
return # Only testing
772+
763773
with SourceLine(initialWorkdir, "listing", WorkflowException, debug):
764774
j.generatefiles["listing"] = ls
765775
for entry in ls:
@@ -824,6 +834,7 @@ def job(
824834
_check_adjust,
825835
)
826836
visit_class([cachebuilder.files, cachebuilder.bindings], ("File"), _checksum)
837+
self._initialworkdir(None, cachebuilder) # test the initial working directory
827838

828839
cmdline = flatten(list(map(cachebuilder.generate_arg, cachebuilder.bindings)))
829840
docker_req, _ = self.get_requirement("DockerRequirement")

‎tests/test_caching.py

+193
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
import re
2+
from pathlib import Path
3+
4+
import pytest
5+
6+
from .util import get_data, get_main_output, needs_docker
7+
8+
test_factors = [(""), ("--parallel"), ("--debug"), ("--parallel --debug")]
9+
10+
11+
@needs_docker
12+
@pytest.mark.parametrize("factor", test_factors)
13+
def test_cid_file_non_existing_dir(tmp_path: Path, factor: str) -> None:
14+
"""Test that --cachedir with a bad path should produce a specific error."""
15+
test_file = "cache_test_workflow.cwl"
16+
bad_cidfile_dir = tmp_path / "cidfile-dir-badpath"
17+
commands = factor.split()
18+
commands.extend(
19+
[
20+
"--record-container-id",
21+
"--cidfile-dir",
22+
str(bad_cidfile_dir),
23+
get_data("tests/wf/" + test_file),
24+
]
25+
)
26+
error_code, _, stderr = get_main_output(commands)
27+
stderr = re.sub(r"\s\s+", " ", stderr)
28+
assert "directory doesn't exist, please create it first" in stderr, stderr
29+
assert error_code == 2 or error_code == 1, stderr
30+
31+
32+
@needs_docker
33+
@pytest.mark.parametrize("factor", test_factors)
34+
def test_wf_without_container(tmp_path: Path, factor: str) -> None:
35+
"""Confirm that we can run a workflow without a container."""
36+
test_file = "hello-workflow.cwl"
37+
cache_dir = str(tmp_path / "cwltool_cache")
38+
commands = factor.split()
39+
commands.extend(
40+
[
41+
"--cachedir",
42+
cache_dir,
43+
"--outdir",
44+
str(tmp_path / "outdir"),
45+
get_data("tests/wf/" + test_file),
46+
"--usermessage",
47+
"hello",
48+
]
49+
)
50+
error_code, _, stderr = get_main_output(commands)
51+
52+
stderr = re.sub(r"\s\s+", " ", stderr)
53+
assert "completed success" in stderr
54+
assert error_code == 0
55+
56+
57+
@needs_docker
58+
@pytest.mark.parametrize("factor", test_factors)
59+
def test_issue_740_fixed(tmp_path: Path, factor: str) -> None:
60+
"""Confirm that re-running a particular workflow with caching succeeds."""
61+
test_file = "cache_test_workflow.cwl"
62+
cache_dir = str(tmp_path / "cwltool_cache")
63+
commands = factor.split()
64+
commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)])
65+
error_code, _, stderr = get_main_output(commands)
66+
67+
stderr = re.sub(r"\s\s+", " ", stderr)
68+
assert "completed success" in stderr
69+
assert error_code == 0
70+
71+
commands = factor.split()
72+
commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)])
73+
error_code, _, stderr = get_main_output(commands)
74+
75+
stderr = re.sub(r"\s\s+", " ", stderr)
76+
assert "Output of job will be cached in" not in stderr
77+
assert error_code == 0, stderr
78+
79+
80+
@needs_docker
81+
@pytest.mark.parametrize("factor", test_factors)
82+
def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
83+
"""Confirm that re-running a particular workflow with caching succeeds."""
84+
test_file = "secondary-files.cwl"
85+
test_job_file = "secondary-files-job.yml"
86+
cache_dir = str(tmp_path / "cwltool_cache")
87+
commands = factor.split()
88+
commands.extend(
89+
[
90+
"--out",
91+
str(tmp_path / "out"),
92+
"--cachedir",
93+
cache_dir,
94+
get_data(f"tests/{test_file}"),
95+
get_data(f"tests/{test_job_file}"),
96+
]
97+
)
98+
error_code, _, stderr = get_main_output(commands)
99+
100+
stderr = re.sub(r"\s\s+", " ", stderr)
101+
assert "completed success" in stderr
102+
assert error_code == 0
103+
104+
commands = factor.split()
105+
commands.extend(
106+
[
107+
"--out",
108+
str(tmp_path / "out2"),
109+
"--cachedir",
110+
cache_dir,
111+
get_data(f"tests/{test_file}"),
112+
get_data(f"tests/{test_job_file}"),
113+
]
114+
)
115+
error_code, _, stderr = get_main_output(commands)
116+
117+
stderr = re.sub(r"\s\s+", " ", stderr)
118+
assert "Output of job will be cached in" not in stderr
119+
assert error_code == 0, stderr
120+
121+
assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists()
122+
123+
124+
@pytest.mark.parametrize("factor", test_factors)
125+
def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None:
126+
"""Confirm that running a CLT with a default literal file with caching succeeds."""
127+
test_file = "tests/wf/extract_region_specs.cwl"
128+
cache_dir = str(tmp_path / "cwltool_cache")
129+
commands = factor.split()
130+
commands.extend(
131+
[
132+
"--out",
133+
str(tmp_path / "out"),
134+
"--cachedir",
135+
cache_dir,
136+
get_data(test_file),
137+
]
138+
)
139+
error_code, _, stderr = get_main_output(commands)
140+
141+
stderr = re.sub(r"\s\s+", " ", stderr)
142+
assert "completed success" in stderr
143+
assert error_code == 0
144+
145+
146+
@needs_docker
147+
@pytest.mark.parametrize("factor", test_factors)
148+
def test_cache_dockerreq_hint_instead_of_req(tmp_path: Path, factor: str) -> None:
149+
"""The cache must not be checked when there is an invalid use of an absolute path in iwdr.listing."""
150+
cache_dir = str(tmp_path / "cwltool_cache")
151+
test_job_file = "tests/wf/loadContents-input.yml"
152+
# First, run the iwd-container-entryname1 conformance tests with caching turned on
153+
test1_file = "tests/wf/iwd-container-entryname1.cwl"
154+
commands1 = factor.split()
155+
commands1.extend(
156+
[
157+
"--out",
158+
str(tmp_path / "out1"),
159+
"--cachedir",
160+
cache_dir,
161+
get_data(test1_file),
162+
get_data(test_job_file),
163+
]
164+
)
165+
error_code1, _, stderr1 = get_main_output(commands1)
166+
167+
stderr1 = re.sub(r"\s\s+", " ", stderr1)
168+
assert "completed success" in stderr1
169+
assert error_code1 == 0
170+
# Second, run the iwd-container-entryname3 test, which should fail
171+
# even though it would be a cache hit, except that its DockerRequirement is
172+
# in `hints` instead of `requirements` and one of the initial working directory
173+
# items has an absolute path starting with `/`.
174+
test2_file = "tests/wf/iwd-container-entryname3.cwl"
175+
commands2 = factor.split()
176+
commands2.extend(
177+
[
178+
"--out",
179+
str(tmp_path / "out2"),
180+
"--cachedir",
181+
cache_dir,
182+
get_data(test2_file),
183+
get_data(test_job_file),
184+
]
185+
)
186+
error_code2, _, stderr2 = get_main_output(commands2)
187+
188+
stderr2 = re.sub(r"\s\s+", " ", stderr2)
189+
assert (
190+
"at index 0 of listing is invalid, name can only start with '/' "
191+
"when DockerRequirement is in 'requirements" in stderr2
192+
)
193+
assert error_code2 == 1

‎tests/test_examples.py

-135
Original file line numberDiff line numberDiff line change
@@ -1115,27 +1115,6 @@ def test_cid_file_dir_arg_is_file_instead_of_dir(tmp_path: Path, factor: str) ->
11151115
assert error_code == 2 or error_code == 1, stderr
11161116

11171117

1118-
@needs_docker
1119-
@pytest.mark.parametrize("factor", test_factors)
1120-
def test_cid_file_non_existing_dir(tmp_path: Path, factor: str) -> None:
1121-
"""Test that --cachedir with a bad path should produce a specific error."""
1122-
test_file = "cache_test_workflow.cwl"
1123-
bad_cidfile_dir = tmp_path / "cidfile-dir-badpath"
1124-
commands = factor.split()
1125-
commands.extend(
1126-
[
1127-
"--record-container-id",
1128-
"--cidfile-dir",
1129-
str(bad_cidfile_dir),
1130-
get_data("tests/wf/" + test_file),
1131-
]
1132-
)
1133-
error_code, _, stderr = get_main_output(commands)
1134-
stderr = re.sub(r"\s\s+", " ", stderr)
1135-
assert "directory doesn't exist, please create it first" in stderr, stderr
1136-
assert error_code == 2 or error_code == 1, stderr
1137-
1138-
11391118
@needs_docker
11401119
@pytest.mark.parametrize("factor", test_factors)
11411120
def test_cid_file_w_prefix(tmp_path: Path, factor: str) -> None:
@@ -1233,120 +1212,6 @@ def test_secondary_files_v1_0(tmp_path: Path, factor: str) -> None:
12331212
assert error_code == 0
12341213

12351214

1236-
@needs_docker
1237-
@pytest.mark.parametrize("factor", test_factors)
1238-
def test_wf_without_container(tmp_path: Path, factor: str) -> None:
1239-
"""Confirm that we can run a workflow without a container."""
1240-
test_file = "hello-workflow.cwl"
1241-
cache_dir = str(tmp_path / "cwltool_cache")
1242-
commands = factor.split()
1243-
commands.extend(
1244-
[
1245-
"--cachedir",
1246-
cache_dir,
1247-
"--outdir",
1248-
str(tmp_path / "outdir"),
1249-
get_data("tests/wf/" + test_file),
1250-
"--usermessage",
1251-
"hello",
1252-
]
1253-
)
1254-
error_code, _, stderr = get_main_output(commands)
1255-
1256-
stderr = re.sub(r"\s\s+", " ", stderr)
1257-
assert "completed success" in stderr
1258-
assert error_code == 0
1259-
1260-
1261-
@needs_docker
1262-
@pytest.mark.parametrize("factor", test_factors)
1263-
def test_issue_740_fixed(tmp_path: Path, factor: str) -> None:
1264-
"""Confirm that re-running a particular workflow with caching succeeds."""
1265-
test_file = "cache_test_workflow.cwl"
1266-
cache_dir = str(tmp_path / "cwltool_cache")
1267-
commands = factor.split()
1268-
commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)])
1269-
error_code, _, stderr = get_main_output(commands)
1270-
1271-
stderr = re.sub(r"\s\s+", " ", stderr)
1272-
assert "completed success" in stderr
1273-
assert error_code == 0
1274-
1275-
commands = factor.split()
1276-
commands.extend(["--cachedir", cache_dir, get_data("tests/wf/" + test_file)])
1277-
error_code, _, stderr = get_main_output(commands)
1278-
1279-
stderr = re.sub(r"\s\s+", " ", stderr)
1280-
assert "Output of job will be cached in" not in stderr
1281-
assert error_code == 0, stderr
1282-
1283-
1284-
@needs_docker
1285-
@pytest.mark.parametrize("factor", test_factors)
1286-
def test_cache_relative_paths(tmp_path: Path, factor: str) -> None:
1287-
"""Confirm that re-running a particular workflow with caching succeeds."""
1288-
test_file = "secondary-files.cwl"
1289-
test_job_file = "secondary-files-job.yml"
1290-
cache_dir = str(tmp_path / "cwltool_cache")
1291-
commands = factor.split()
1292-
commands.extend(
1293-
[
1294-
"--out",
1295-
str(tmp_path / "out"),
1296-
"--cachedir",
1297-
cache_dir,
1298-
get_data(f"tests/{test_file}"),
1299-
get_data(f"tests/{test_job_file}"),
1300-
]
1301-
)
1302-
error_code, _, stderr = get_main_output(commands)
1303-
1304-
stderr = re.sub(r"\s\s+", " ", stderr)
1305-
assert "completed success" in stderr
1306-
assert error_code == 0
1307-
1308-
commands = factor.split()
1309-
commands.extend(
1310-
[
1311-
"--out",
1312-
str(tmp_path / "out2"),
1313-
"--cachedir",
1314-
cache_dir,
1315-
get_data(f"tests/{test_file}"),
1316-
get_data(f"tests/{test_job_file}"),
1317-
]
1318-
)
1319-
error_code, _, stderr = get_main_output(commands)
1320-
1321-
stderr = re.sub(r"\s\s+", " ", stderr)
1322-
assert "Output of job will be cached in" not in stderr
1323-
assert error_code == 0, stderr
1324-
1325-
assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists()
1326-
1327-
1328-
@pytest.mark.parametrize("factor", test_factors)
1329-
def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None:
1330-
"""Confirm that running a CLT with a default literal file with caching succeeds."""
1331-
test_file = "tests/wf/extract_region_specs.cwl"
1332-
cache_dir = str(tmp_path / "cwltool_cache")
1333-
commands = factor.split()
1334-
commands.extend(
1335-
[
1336-
"--out",
1337-
str(tmp_path / "out"),
1338-
"--cachedir",
1339-
cache_dir,
1340-
get_data(test_file),
1341-
]
1342-
)
1343-
error_code, _, stderr = get_main_output(commands)
1344-
1345-
stderr = re.sub(r"\s\s+", " ", stderr)
1346-
assert "completed success" in stderr
1347-
assert error_code == 0
1348-
1349-
13501215
def test_write_summary(tmp_path: Path) -> None:
13511216
"""Test --write-summary."""
13521217
commands = [

0 commit comments

Comments
 (0)