galaxy-tool-util: use a better location for the container_image_cache_path

Author: mr-c

cwltool/main.py

@@ -1289,6 +1289,7 @@ def main(
                     find_default_container,
                     default_container=runtimeContext.default_container,
                     use_biocontainers=args.beta_use_biocontainers,
+                    container_image_cache_path=args.beta_dependencies_directory,
                 )
 
             (out, status) = real_executor(
@@ -1410,10 +1411,13 @@ def find_default_container(
     builder: HasReqsHints,
     default_container: Optional[str] = None,
     use_biocontainers: Optional[bool] = None,
+    container_image_cache_path: Optional[str] = None,
 ) -> Optional[str]:
     """Find a container."""
     if not default_container and use_biocontainers:
-        default_container = get_container_from_software_requirements(use_biocontainers, builder)
+        default_container = get_container_from_software_requirements(
+            use_biocontainers, builder, container_image_cache_path
+        )
     return default_container
 
 

cwltool/software_requirements.py

@@ -136,7 +136,7 @@ def get_dependencies(builder: HasReqsHints) -> ToolRequirements:
 
 
 def get_container_from_software_requirements(
-    use_biocontainers: bool, builder: HasReqsHints
+    use_biocontainers: bool, builder: HasReqsHints, container_image_cache_path: Optional[str] = "."
 ) -> Optional[str]:
     if use_biocontainers:
         ensure_galaxy_lib_available()
@@ -147,7 +147,7 @@ def get_container_from_software_requirements(
         app_info: AppInfo = AppInfo(
             involucro_auto_init=True,
             enable_mulled_containers=True,
-            container_image_cache_path=".",
+            container_image_cache_path=container_image_cache_path,
         )
         container_registry: ContainerRegistry = ContainerRegistry(app_info)
         requirements = get_dependencies(builder)

tests/test_dependencies.py

@@ -1,10 +1,11 @@
 """Tests of satisfying SoftwareRequirement via dependencies."""
 import os
+import tempfile
 from pathlib import Path
 from shutil import which
 from types import ModuleType
-from typing import Optional
-
+from typing import Optional, Tuple, Callable
+from getpass import getuser
 import pytest
 
 from cwltool.context import LoadingContext
@@ -26,7 +27,15 @@ def test_biocontainers(tmp_path: Path) -> None:
     wflow = get_data("tests/seqtk_seq.cwl")
     job = get_data("tests/seqtk_seq_job.json")
     error_code, _, _ = get_main_output(
-        ["--outdir", str(tmp_path), "--beta-use-biocontainers", wflow, job]
+        [
+            "--outdir",
+            str(tmp_path / "out"),
+            "--beta-use-biocontainers",
+            "--beta-dependencies-directory",
+            str(tmp_path / "deps"),
+            wflow,
+            job,
+        ]
     )
 
     assert error_code == 0
@@ -38,31 +47,84 @@ def test_biocontainers_resolution(tmp_path: Path) -> None:
     """Confirm expected container name for --beta-use-biocontainers."""
     tool = load_tool(get_data("tests/seqtk_seq.cwl"), LoadingContext())
     assert (
-        get_container_from_software_requirements(True, tool) == "quay.io/biocontainers/seqtk:r93--0"
+        get_container_from_software_requirements(
+            True, tool, container_image_cache_path=str(tmp_path)
+        )
+        == "quay.io/biocontainers/seqtk:r93--0"
     )
 
 
-@pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed")
-def test_bioconda(tmp_path: Path) -> None:
+@pytest.fixture(scope="session")
+def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]:
+    """
+    Caches the conda environment created for seqtk_seq.cwl.
+
+    Respects ``--basetemp`` via code copied from
+    :py:method:`pytest.TempPathFactory.getbasetemp`.
+    """
+
+    deps_dir = request.config.cache.get("bioconda_deps", None)
+    if deps_dir is not None and not Path(deps_dir).exists():
+        # cache value set, but cache is gone :( ... recreate
+        deps_dir = None
+
+    if deps_dir is None:
+        given_basetemp = request.config.option.basetemp
+        if given_basetemp is not None:
+            basetemp = Path(os.path.abspath(str(given_basetemp))).resolve()
+            deps_dir = basetemp / "bioconda"
+        else:
+            from_env = os.environ.get("PYTEST_DEBUG_TEMPROOT")
+            temproot = Path(from_env or tempfile.gettempdir()).resolve()
+            rootdir = temproot.joinpath(f"pytest-of-{getuser() or 'unknown'}")
+            try:
+                rootdir.mkdir(mode=0o700, exist_ok=True)
+            except OSError:
+                rootdir = temproot.joinpath("pytest-of-unknown")
+                rootdir.mkdir(mode=0o700, exist_ok=True)
+            deps_dir = rootdir / "bioconda"
+        request.config.cache.set("bioconda_deps", str(deps_dir))
+
+    deps_dirpath = Path(deps_dir)
+    deps_dirpath.mkdir(exist_ok=True)
+
     wflow = get_data("tests/seqtk_seq.cwl")
     job = get_data("tests/seqtk_seq_job.json")
     error_code, _, stderr = get_main_output(
-        ["--outdir", str(tmp_path), "--beta-conda-dependencies", "--debug", wflow, job]
+        [
+            "--outdir",
+            str(deps_dirpath / "out"),
+            "--beta-conda-dependencies",
+            "--beta-dependencies-directory",
+            str(deps_dirpath / "deps"),
+            "--debug",
+            wflow,
+            job,
+        ]
     )
+    return error_code, stderr
 
+
+@pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed")
+def test_bioconda(bioconda_setup: Callable[[], Tuple[Optional[int], str]]) -> None:
+    error_code, stderr = bioconda_setup
     assert error_code == 0, stderr
 
 
 @pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed")
 @pytest.mark.skipif(not which("modulecmd"), reason="modulecmd not installed")
-def test_modules(monkeypatch: pytest.MonkeyPatch) -> None:
+def test_modules(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
     """Do a basic smoke test using environment modules to satisfy a SoftwareRequirement."""
     wflow = get_data("tests/random_lines.cwl")
     job = get_data("tests/random_lines_job.json")
     monkeypatch.setenv("MODULEPATH", os.path.join(os.getcwd(), "tests/test_deps_env/modulefiles"))
     error_code, _, stderr = get_main_output(
         [
+            "--outdir",
+            str(tmp_path / "out"),
             "--beta-dependency-resolvers-configuration",
+            "--beta-dependencies-directory",
+            str(tmp_path / "deps"),
             "tests/test_deps_env_modules_resolvers_conf.yml",
             "--debug",
             wflow,