Skip to content

Commit ccfa0d9

Browse files
jmgatejmgate
committed
test: Run the examples and check their output
1 parent 2207516 commit ccfa0d9

11 files changed

+341
-2
lines changed

‎.github/workflows/continuous-integration.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ jobs:
4040
run: python3 -m pip install .
4141

4242
- name: Test with pytest
43-
run: python3 -m pytest --cov=staged_script test/
43+
run: python3 -m pytest --cov=staged_script example/ test/
4444

4545
- name: Upload coverage reports to Codecov
4646
uses: codecov/codecov-action@v4

‎example/ex_0_the_basics.py

100644100755
+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/usr/bin/env python3
12
"""A very basic staged script."""
23

34
# © 2024 National Technology & Engineering Solutions of Sandia, LLC

‎example/ex_1_removing_the_retry_arguments.py

100644100755
+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/usr/bin/env python3
12
"""A staged script without retry arguments."""
23

34
# © 2024 National Technology & Engineering Solutions of Sandia, LLC

‎example/ex_2_running_certain_stages_by_default.py

100644100755
+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/usr/bin/env python3
12
"""A staged script that runs certain stages by default."""
23

34
# © 2024 National Technology & Engineering Solutions of Sandia, LLC

‎example/ex_3_adding_arguments.py

100644100755
+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/usr/bin/env python3
12
"""A staged script with additional arguments."""
23

34
# © 2024 National Technology & Engineering Solutions of Sandia, LLC

‎example/ex_4_customizing_stage_behavior.py

100644100755
+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/usr/bin/env python3
12
"""A staged script with custom stage behavior."""
23

34
# © 2024 National Technology & Engineering Solutions of Sandia, LLC

‎example/ex_5_customizing_individual_stages.py

100644100755
+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/usr/bin/env python3
12
"""A staged script with phases customized per stage."""
23

34
# © 2024 National Technology & Engineering Solutions of Sandia, LLC

‎example/ex_6_creating_retryable_stages.py

100644100755
+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/usr/bin/env python3
12
"""A staged script with a retryable stage."""
23

34
# © 2024 National Technology & Engineering Solutions of Sandia, LLC

‎example/ex_7_customizing_the_summary.py

100644100755
+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#!/usr/bin/env python3
12
"""A staged script with a retryable stage."""
23

34
# © 2024 National Technology & Engineering Solutions of Sandia, LLC

‎example/test_examples.py

+329
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,329 @@
1+
#!/usr/bin/env python3
2+
"""Run all the examples and ensure their output is correct."""
3+
4+
# © 2024 National Technology & Engineering Solutions of Sandia, LLC
5+
# (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the
6+
# U.S. Government retains certain rights in this software.
7+
8+
# SPDX-License-Identifier: BSD-3-Clause
9+
10+
import subprocess
11+
from pathlib import Path
12+
from typing import List
13+
14+
15+
def assert_output_in_order(stdout: str, output: List[str]) -> None:
16+
"""
17+
Ensure the output appears in the correct order.
18+
19+
Args:
20+
stdout: The ``stdout`` of the command that was run.
21+
output: The list of terms that should appear in the output in
22+
the given order.
23+
24+
Raises:
25+
ValueError: If any of the terms in the output list cannot be
26+
found.
27+
"""
28+
index = 0
29+
for term in output:
30+
index = stdout.find(term, index) + len(term)
31+
32+
33+
def test_ex_0_the_basics_help() -> None:
34+
example = Path(__file__).parent / "ex_0_the_basics.py"
35+
result = subprocess.run(
36+
[example, "--help"],
37+
capture_output=True,
38+
check=True,
39+
text=True,
40+
)
41+
assert_output_in_order(
42+
result.stdout,
43+
[
44+
"the ArgumentParser for the StagedScript base class.",
45+
"--stage {goodbye,hello}",
46+
"--goodbye-retry-attempts",
47+
"--goodbye-retry-delay",
48+
"--goodbye-retry-timeout",
49+
"--hello-retry-attempts",
50+
"--hello-retry-delay",
51+
"--hello-retry-timeout",
52+
],
53+
)
54+
55+
56+
def test_ex_0_the_basics() -> None:
57+
example = Path(__file__).parent / "ex_0_the_basics.py"
58+
result = subprocess.run(
59+
[example, "--stage", "hello"],
60+
capture_output=True,
61+
check=True,
62+
text=True,
63+
)
64+
assert_output_in_order(
65+
result.stdout,
66+
[
67+
"Greeting the user",
68+
"Executing: echo 'Hello World'",
69+
"`hello` stage duration",
70+
"Bidding farewell",
71+
"Skipping this stage",
72+
"`goodbye` stage duration",
73+
"Ran the following",
74+
"--hello-retry-attempts 0",
75+
"--hello-retry-delay 0",
76+
"--hello-retry-timeout 60",
77+
"--goodbye-retry-attempts 0",
78+
"--goodbye-retry-delay 0",
79+
"--goodbye-retry-timeout 60",
80+
"Commands executed",
81+
"Timing results",
82+
"Script result",
83+
],
84+
)
85+
86+
87+
def test_ex_1_removing_the_retry_arguments_help() -> None:
88+
example = Path(__file__).parent / "ex_1_removing_the_retry_arguments.py"
89+
result = subprocess.run(
90+
[example, "--help"],
91+
capture_output=True,
92+
check=True,
93+
text=True,
94+
)
95+
assert "Demonstrate removing the retry arguments" in result.stdout
96+
for _ in [
97+
"--goodbye-retry-attempts",
98+
"--goodbye-retry-delay",
99+
"--goodbye-retry-timeout",
100+
"--hello-retry-attempts",
101+
"--hello-retry-delay",
102+
"--hello-retry-timeout",
103+
]:
104+
assert _ not in result.stdout
105+
106+
107+
def test_ex_1_removing_the_retry_arguments() -> None:
108+
example = Path(__file__).parent / "ex_1_removing_the_retry_arguments.py"
109+
result = subprocess.run(
110+
[example, "--stage", "hello"],
111+
capture_output=True,
112+
check=True,
113+
text=True,
114+
)
115+
for _ in [
116+
"--goodbye-retry-attempts",
117+
"--goodbye-retry-delay",
118+
"--goodbye-retry-timeout",
119+
"--hello-retry-attempts",
120+
"--hello-retry-delay",
121+
"--hello-retry-timeout",
122+
]:
123+
assert _ not in result.stdout
124+
125+
126+
def test_ex_2_running_certain_stages_by_default() -> None:
127+
example = (
128+
Path(__file__).parent / "ex_2_running_certain_stages_by_default.py"
129+
)
130+
result = subprocess.run(
131+
[example],
132+
capture_output=True,
133+
check=True,
134+
text=True,
135+
)
136+
assert_output_in_order(
137+
result.stdout,
138+
[
139+
"Greeting the user",
140+
"Executing: echo 'Hello World'",
141+
"`hello` stage duration",
142+
"Bidding farewell",
143+
"Executing: echo 'Goodbye World'",
144+
"`goodbye` stage duration",
145+
],
146+
)
147+
148+
149+
def test_ex_3_adding_arguments() -> None:
150+
example = Path(__file__).parent / "ex_3_adding_arguments.py"
151+
result = subprocess.run(
152+
[example, "--some-file", "foo.txt"],
153+
capture_output=True,
154+
check=True,
155+
text=True,
156+
)
157+
assert_output_in_order(
158+
result.stdout,
159+
[
160+
"Greeting the user",
161+
"Executing: echo 'Hello World'",
162+
"Processing file",
163+
"/foo.txt",
164+
"`hello` stage duration",
165+
"Bidding farewell",
166+
"Executing: echo 'Goodbye World'",
167+
"Some flag was not set!",
168+
"`goodbye` stage duration",
169+
],
170+
)
171+
172+
173+
def test_ex_4_customizing_stage_behavior() -> None:
174+
example = Path(__file__).parent / "ex_4_customizing_stage_behavior.py"
175+
result = subprocess.run(
176+
[
177+
example,
178+
"--some-file",
179+
"foo.txt",
180+
"--some-flag",
181+
"--stage",
182+
"goodbye",
183+
],
184+
capture_output=True,
185+
check=True,
186+
text=True,
187+
)
188+
assert_output_in_order(
189+
result.stdout,
190+
[
191+
"Checking to make sure it's safe to run a stage",
192+
"Greeting the user",
193+
"You didn't tell me to run the 'hello' stage",
194+
"`hello` stage duration",
195+
"Finished stage 'hello'",
196+
"Checking to make sure all is well after running",
197+
"Checking to make sure it's safe to run a stage",
198+
"Bidding farewell",
199+
"Executing: echo 'Goodbye World'",
200+
"Some flag was set!",
201+
"`goodbye` stage duration",
202+
"Finished stage 'goodbye'",
203+
"Checking to make sure all is well after running",
204+
],
205+
)
206+
207+
208+
def test_ex_5_customizing_individual_stages() -> None:
209+
example = Path(__file__).parent / "ex_5_customizing_individual_stages.py"
210+
result = subprocess.run(
211+
[example, "--some-file", "foo.txt"],
212+
capture_output=True,
213+
check=True,
214+
text=True,
215+
)
216+
assert_output_in_order(
217+
result.stdout,
218+
[
219+
"Checking to make sure it's safe to run a stage",
220+
"Greeting the user",
221+
"The first stage is underway",
222+
"Executing: echo 'Hello World'",
223+
"Processing file",
224+
"/foo.txt",
225+
"`hello` stage duration",
226+
"Finished stage 'hello'",
227+
"Checking to make sure all is well after running",
228+
"Checking to make sure it's safe to run a stage",
229+
"Bidding farewell",
230+
"Executing: echo 'Goodbye World'",
231+
"Some flag was not set!",
232+
"`goodbye` stage duration",
233+
"Finished the final stage: goodbye",
234+
"Checking to make sure all is well after running",
235+
],
236+
)
237+
238+
239+
def test_ex_6_creating_retryable_stages_help() -> None:
240+
example = Path(__file__).parent / "ex_6_creating_retryable_stages.py"
241+
result = subprocess.run(
242+
[example, "--help"],
243+
capture_output=True,
244+
check=True,
245+
text=True,
246+
)
247+
assert_output_in_order(
248+
result.stdout,
249+
[
250+
"retry",
251+
"Additional options for retrying stages",
252+
"--flaky-retry-attempts",
253+
"--flaky-retry-delay",
254+
"--flaky-retry-timeout",
255+
],
256+
)
257+
258+
259+
def test_ex_6_creating_retryable_stages() -> None:
260+
example = Path(__file__).parent / "ex_6_creating_retryable_stages.py"
261+
result = subprocess.run(
262+
[example, "--some-file", "foo.txt"],
263+
capture_output=True,
264+
check=True,
265+
text=True,
266+
)
267+
flaky_output = [
268+
"Trying an error-prone operation",
269+
"Oh no! Something went horribly wrong!",
270+
"`flaky` stage duration",
271+
"Finished stage 'flaky'",
272+
"Preparing to retry the 'flaky' stage",
273+
"RetryCallState",
274+
]
275+
assert_output_in_order(
276+
result.stdout,
277+
[
278+
*flaky_output,
279+
*flaky_output,
280+
"Trying an error-prone operation",
281+
"Thank goodness, everything worked this time",
282+
"`flaky` stage duration",
283+
"Finished stage 'flaky'",
284+
],
285+
)
286+
287+
288+
def test_ex_6_creating_retryable_stages_no_retries() -> None:
289+
example = Path(__file__).parent / "ex_6_creating_retryable_stages.py"
290+
result = subprocess.run(
291+
[example, "--some-file", "foo.txt", "--flaky-retry-attempts", "0"],
292+
capture_output=True,
293+
check=True,
294+
text=True,
295+
)
296+
assert_output_in_order(
297+
result.stdout,
298+
[
299+
"Trying an error-prone operation",
300+
"Oh no! Something went horribly wrong!",
301+
"`flaky` stage duration",
302+
"Finished stage 'flaky'",
303+
"Checking to make sure all is well after running",
304+
"Checking to make sure it's safe to run a stage",
305+
"Bidding farewell",
306+
"Script result",
307+
"Failure",
308+
],
309+
)
310+
311+
312+
def test_ex_7_customizing_the_summary() -> None:
313+
example = Path(__file__).parent / "ex_7_customizing_the_summary.py"
314+
result = subprocess.run(
315+
[example, "--some-file", "foo.txt", "--stage", "hello"],
316+
capture_output=True,
317+
check=True,
318+
text=True,
319+
)
320+
assert_output_in_order(
321+
result.stdout,
322+
[
323+
"Script result",
324+
"Success",
325+
"Machine details",
326+
"hostname",
327+
"platform",
328+
],
329+
)

‎pyproject.toml

+3-1
Original file line numberDiff line numberDiff line change
@@ -104,10 +104,12 @@ ignore = [
104104

105105
[tool.ruff.lint.per-file-ignores]
106106
"**/test_*.py" = ["S101"]
107-
"example/ex_*.py" = [
107+
"example/*.py" = [
108108
"D101",
109109
"D102",
110+
"D103",
110111
"D107",
112+
"S603",
111113
"S604",
112114
]
113115

0 commit comments

Comments
 (0)