BatchJob.start_and_wait: add require_success argument

related to ESA-APEx/apex_algorithms#163

Author: soxofaan

CHANGELOG.md

@@ -12,10 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `openeo.testing.io.TestDataLoader`: unit test utility to compactly load (and optionally preprocess) tests data (text/JSON/...)
 - `openeo.Connection`: automatically retry API requests on `429 Too Many Requests` HTTP errors, with appropriate delay if possible ([#441](https://github.com/Open-EO/openeo-python-client/issues/441))
 - Introduced `pixel_tolerance` argument in `openeo.testing.results` helpers to specify the ignorable fraction of significantly differing pixels. ([#776](https://github.com/Open-EO/openeo-python-client/issues/776))
+- `BatchJob.start_and_wait()`: add `require_success` argument (on by default) to control whether an exception should be raised automatically on job failure.
 
 ### Changed
 
 - `DataCube.apply_dimension()`: not explicitly specifying the `dimension` argument is deprecated and will trigger warnings ([#774](https://github.com/Open-EO/openeo-python-client/issues/774))
+- `BatchJob.start_and_wait()`: all arguments must be specified as keyword arguments to eliminate the risk of positional mix-ups between all its heterogeneous arguments and flags.
 
 ### Removed
 

openeo/rest/job.py

@@ -273,11 +273,13 @@ def run_synchronous(
 
     def start_and_wait(
         self,
+        *,
         print=print,
         max_poll_interval: float = DEFAULT_JOB_STATUS_POLL_INTERVAL_MAX,
         connection_retry_interval: float = DEFAULT_JOB_STATUS_POLL_CONNECTION_RETRY_INTERVAL,
         soft_error_max: int = DEFAULT_JOB_STATUS_POLL_SOFT_ERROR_MAX,
         show_error_logs: bool = True,
+        require_success: bool = True,
     ) -> BatchJob:
         """
         Start the batch job, poll its status and wait till it finishes (or fails)
@@ -287,11 +289,19 @@ def start_and_wait(
         :param connection_retry_interval: how long to wait when status poll failed due to connection issue
         :param soft_error_max: maximum number of soft errors (e.g. temporary connection glitches) to allow
         :param show_error_logs: whether to automatically print error logs when the batch job failed.
+        :param require_success: whether to raise an exception if the job did not finish successfully.
 
         :return: Handle to the job created at the backend.
 
         .. versionchanged:: 0.37.0
             Added argument ``show_error_logs``.
+
+        .. versionchanged:: 0.42.0
+            All arguments must be specified as keyword arguments,
+            to eliminate the risk of positional mix-ups between heterogeneous arguments and flags.
+
+        .. versionchanged:: 0.42.0
+            Added argument ``require_success``.
         """
         # TODO rename `connection_retry_interval` to something more generic?
         start_time = time.time()
@@ -354,7 +364,7 @@ def soft_error(message: str):
             time.sleep(poll_interval)
             poll_interval = min(1.25 * poll_interval, max_poll_interval)
 
-        if status != "finished":
+        if require_success and status != "finished":
             # TODO: render logs jupyter-aware in a notebook context?
             if show_error_logs:
                 print(f"Your batch job {self.job_id!r} failed. Error logs:")

tests/rest/test_job.py

@@ -3,6 +3,7 @@
 import json
 import logging
 import re
+from contextlib import nullcontext
 from pathlib import Path
 from typing import Optional
 from unittest import mock
@@ -321,6 +322,23 @@ def test_execute_batch_with_excessive_soft_errors(con100, requests_mock, tmpdir,
     ]
 
 
+@pytest.mark.parametrize(
+    ["require_success", "expectation"],
+    [
+        (True, pytest.raises(JobFailedException, match="'job-000' didn't finish successfully")),
+        (False, contextlib.nullcontext()),
+    ],
+)
+def test_start_and_wait_with_error_require_success(dummy_backend, require_success, expectation):
+    dummy_backend.setup_simple_job_status_flow(queued=0, running=1, final="error")
+    cube = dummy_backend.connection.load_collection("S2").save_result(format="GTiff")
+    job = cube.create_job()
+    assert job.status() == "created"
+    with expectation, fake_time():
+        job.start_and_wait(require_success=require_success)
+    assert job.status() == "error"
+
+
 @httpretty.activate(allow_net_connect=False)
 @pytest.mark.parametrize(
     ["retry_config", "extra_responses", "expectation_context", "expected_sleeps"],