DM-51000 hotfix 1

doc/changes/DM-51000.misc.rst

@@ -0,0 +1 @@
+Updated the local submissions to handle the case when the given executable name is not an absolute path.  It uses the submit-side location of the executable or prints a warning that the job may be held.  This was triggered by ``ctrl_bps`` default yaml definitions no longer using ``*_DIR`` environment variables; instead they now rely on the PATH.

python/lsst/ctrl/bps/htcondor/prepare_utils.py

@@ -30,6 +30,7 @@
 import logging
 import os
 import re
+import shutil
 from collections import defaultdict
 from pathlib import Path
 from typing import Any, cast
@@ -283,6 +284,50 @@ def _translate_job_cmds(cached_vals, generic_workflow, gwjob):
     return jobcmds
 
 
+def _resolve_executable(executable: str) -> str:
+    """Resolve an executable name to its full path via PATH.
+
+    HTCondor resolves a relative ``executable`` in the submit description
+    relative to the job's sandbox directory rather than searching ``PATH``.
+    So a bare or relative command name (e.g., ``pipetask``) must be turned
+    into a full path at submit time, otherwise the job is held with a
+    "No such file or directory" error on the compute node.
+
+    Parameters
+    ----------
+    executable : `str`
+        The executable value from the job description.
+
+    Returns
+    -------
+    executable : `str`
+        A full path if ``executable`` was a bare name found on ``PATH``,
+        otherwise the value unchanged (already absolute/relative paths and
+        values containing HTCondor env syntax are left as is).
+    """
+    # Leave anything that already starts with a path separator or
+    # environment-variable syntax (e.g., $ENV(...), $(...)) untouched.
+    if executable.startswith(("/", "$")):
+        _LOG.debug("Executable started with / or $.  Using as-is (%s)", executable)
+        return executable
+
+    exec_path = shutil.which(executable)
+    if exec_path:
+        _LOG.debug("results of 'which %s' = %s", executable, exec_path)
+        # Still have to make sure it is not a relative path as
+        # shutil.which would skip checking PATH if executable
+        # contains '/' and then only check existence.
+        exec_path = os.path.abspath(exec_path)
+        return exec_path
+
+    _LOG.warning(
+        "Could not resolve executable '%s' on PATH; HTCondor resolves an "
+        "executable name relative to the job sandbox, so the job may be held.",
+        executable,
+    )
+    return executable
+
+
 def _translate_command_line(
     cached_vals: dict[str, Any], generic_workflow: GenericWorkflow, gwjob: GenericWorkflowJob
 ) -> dict[str, Any]:
@@ -353,7 +398,7 @@ def _translate_command_line(
             jobcmds["transfer_executable"] = "True"
             jobcmds["executable"] = gwjob.executable.src_uri
         else:
-            jobcmds["executable"] = _fix_env_var_syntax(gwjob.executable.src_uri)
+            jobcmds["executable"] = _resolve_executable(_fix_env_var_syntax(gwjob.executable.src_uri))
 
         if arguments:
             arguments = _fix_env_var_syntax(arguments)

tests/test_prepare_utils.py

@@ -29,6 +29,7 @@
 
 import logging
 import os
+import shutil
 import unittest
 from copy import deepcopy
 
@@ -222,6 +223,26 @@ def testMakeCommandExecutableEnvVar(self):
         jobcmds = prepare_utils._translate_command_line(self.cached_vals, gw, gwjob)
         self.assertEqual(jobcmds["executable"], "$ENV(CTRL_BPS_DIR)/bin/pipetask")
 
+    def testMakeCommandBareExecutableResolvedViaPath(self):
+        # A bare executable name (no path, no env var) is resolved to its
+        # full path via PATH because HTCondor resolves the executable
+        # relative to the job sandbox, not via PATH.
+        resolved = shutil.which("sh")
+        self.assertIsNotNone(resolved, "test requires 'sh' on PATH")
+        gw_exec = GenericWorkflowExec("sh", "sh")
+        gw, gwjob = self._make_job(executable=gw_exec)
+        jobcmds = prepare_utils._translate_command_line(self.cached_vals, gw, gwjob)
+        self.assertEqual(jobcmds["executable"], resolved)
+
+    def testMakeCommandBareExecutableNotOnPath(self):
+        # A bare executable name that cannot be found on PATH is left as is.
+        gw_exec = GenericWorkflowExec("nosuchcmd", "nosuchcmd_xyzzy")
+        gw, gwjob = self._make_job(executable=gw_exec)
+        with self.assertLogs(level="WARNING") as cm_log:
+            jobcmds = prepare_utils._translate_command_line(self.cached_vals, gw, gwjob)
+        self.assertRegex(cm_log.output[0], "Could not resolve executable 'nosuchcmd_xyzzy' on PATH.*")
+        self.assertEqual(jobcmds["executable"], "nosuchcmd_xyzzy")
+
     def testMakeCommandArguments(self):
         # Arguments should have cmd, wms, file, and env placeholders replaced.
         gw, gwjob = self._make_job(arguments="run <FILE:qg> --attempt <WMS:attemptNum> {opt}")