Extended container support

planemo/galaxy/config.py

@@ -26,9 +26,13 @@
     TYPE_CHECKING,
 )
 
-from galaxy.tool_util.deps import docker_util
+from galaxy.tool_util.deps import (
+    docker_util,
+    singularity_util,
+)
 from galaxy.tool_util.deps.container_volumes import DockerVolume
 from galaxy.util.commands import argv_to_str
+from galaxy.util.yaml_util import ordered_dump
 from packaging.version import parse as parse_version
 
 from planemo import git
@@ -122,32 +126,23 @@
 </tool_sheds>
 """
 
-JOB_CONFIG_LOCAL = """<job_conf>
-    <plugins>
-        <plugin id="planemo_runner" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="4"/>
-    </plugins>
-    <handlers>
-    </handlers>
-    <destinations default="planemo_dest">
-        <destination id="planemo_dest" runner="planemo_runner">
-            <param id="require_container">${require_container}</param>
-            <param id="docker_enabled">${docker_enable}</param>
-            <param id="docker_sudo">${docker_sudo}</param>
-            <param id="docker_sudo_cmd">${docker_sudo_cmd}</param>
-            <param id="docker_cmd">${docker_cmd}</param>
-            <param id="docker_volumes">${docker_volumes}</param>
-            <param id="docker_run_extra_arguments"><![CDATA[${docker_run_extra_arguments}]]></param>
-            ${docker_host_param}
-        </destination>
-        <destination id="upload_dest" runner="planemo_runner">
-            <param id="docker_enabled">false</param>
-        </destination>
-    </destinations>
-    <tools>
-        <tool id="upload1" destination="upload_dest" />
-    </tools>
-</job_conf>
-"""
+JOB_CONFIG_LOCAL: Dict[str, Any] = {
+    "runners": {"planemo_runner": {"load": "galaxy.jobs.runners.local:LocalJobRunner", "workers": 4}},
+    "execution": {
+        "default": "planemo_dest",
+        "environments": {
+            "planemo_dest": {
+                "runner": "planemo_runner",
+                "require_container": False,
+            },
+            "upload_dest": {"runner": "planemo_runner", "docker_enabled": False},
+        },
+    },
+    "tools": [
+        {"id": "upload1", "environment": "upload_dest"},
+    ],
+}
+
 
 REFGENIE_CONFIG_TEMPLATE = """
 config_version: %s
@@ -345,11 +340,14 @@ def local_galaxy_config(ctx, runnables, for_tests=False, **kwds):
 
     # Duplicate block in docker variant above.
     if kwds.get("mulled_containers", False):
-        if not kwds.get("docker", False):
-            if ctx.get_option_source("docker") != OptionSource.cli:
+        if not (kwds.get("docker", False) or kwds.get("singularity", False)):
+            if (
+                ctx.get_option_source("docker") != OptionSource.cli
+                and ctx.get_option_source("singularity") != OptionSource.cli
+            ):
                 kwds["docker"] = True
             else:
-                raise Exception("Specified no docker and mulled containers together.")
+                raise Exception("Specified --no-docker/--no-singularity and mulled containers together.")
         conda_default_options = ("conda_auto_init", "conda_auto_install")
         use_conda_options = ("dependency_resolution", "conda_use_local", "conda_prefix", "conda_exec")
         if not any(kwds.get(_) for _ in use_conda_options) and all(
@@ -740,7 +738,7 @@ class GalaxyConfig(GalaxyInterface, metaclass=abc.ABCMeta):
 
     This assumes more than an API connection is available - Planemo needs to be able to
     start and stop the Galaxy instance, recover logs, etc... There are currently two
-    implementations - a locally executed Galaxy and one running inside a Docker containe
+    implementations - a locally executed Galaxy and one running inside a Docker container
     """
 
     @abc.abstractproperty
@@ -1376,42 +1374,51 @@ def _handle_job_config_file(
 ):
     job_config_file = kwds.get("job_config_file", None)
     if not job_config_file:
-        template_str = JOB_CONFIG_LOCAL
         job_config_file = os.path.join(
             config_directory,
-            "job_conf.xml",
+            "job_conf.yml",
         )
-        docker_enable = str(kwds.get("docker", False))
-        docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST)
-        docker_host_param = ""
-        if docker_host:
-            docker_host_param = f"""<param id="docker_host">{docker_host}</param>"""
-
-        volumes = list(kwds.get("docker_extra_volume") or [])
-        if test_data_dir:
-            volumes.append(f"{test_data_dir}:ro")
-
-        docker_volumes_str = "$defaults"
-        if volumes:
-            # exclude tool directories, these are mounted :ro by $defaults
-            all_tool_dirs = {os.path.dirname(tool_path) for tool_path in all_tool_paths}
-            extra_volumes_str = ",".join(str(v) for v in create_docker_volumes(volumes) if v.path not in all_tool_dirs)
-            docker_volumes_str = f"{docker_volumes_str},{extra_volumes_str}"
-
-        conf_contents = Template(template_str).safe_substitute(
-            {
-                "server_name": server_name,
-                "docker_enable": docker_enable,
-                "require_container": "false",
-                "docker_sudo": str(kwds.get("docker_sudo", False)),
-                "docker_sudo_cmd": str(kwds.get("docker_sudo_cmd", docker_util.DEFAULT_SUDO_COMMAND)),
-                "docker_cmd": str(kwds.get("docker_cmd", docker_util.DEFAULT_DOCKER_COMMAND)),
-                "docker_host_param": docker_host_param,
-                "docker_volumes": docker_volumes_str,
-                "docker_run_extra_arguments": kwds.get("docker_run_extra_arguments", ""),
-            }
-        )
-        write_file(job_config_file, conf_contents)
+        planemo_dest = JOB_CONFIG_LOCAL["execution"]["environments"]["planemo_dest"]
+
+        for container_type in ["docker", "singularity"]:
+            if not kwds.get(container_type, False):
+                continue
+            planemo_dest[f"{container_type}_enabled"] = kwds.get(container_type, False)
+            planemo_dest[f"{container_type}_sudo"] = kwds.get(f"{container_type}_sudo", False)
+            planemo_dest[f"{container_type}_sudo_cmd"] = kwds.get(
+                f"{container_type}_sudo_cmd",
+                docker_util.DEFAULT_SUDO_COMMAND
+                if container_type == "docker"
+                else singularity_util.DEFAULT_SUDO_COMMAND,
+            )
+            planemo_dest[f"{container_type}_cmd"] = kwds.get(
+                f"{container_type}_cmd",
+                docker_util.DEFAULT_DOCKER_COMMAND
+                if container_type == "docker"
+                else singularity_util.DEFAULT_SINGULARITY_COMMAND,
+            )
+            if container_type == "docker":
+                docker_host = kwds.get("docker_host", docker_util.DEFAULT_HOST)
+                if docker_host:
+                    planemo_dest["docker_host"] = docker_host
+
+            volumes = list(kwds.get(f"{container_type}_extra_volume") or [])
+            if test_data_dir:
+                volumes.append(f"{test_data_dir}:ro")
+            volumes_str = "$defaults"
+            if volumes:
+                # exclude tool directories, these are mounted :ro by $defaults
+                all_tool_dirs = {os.path.dirname(tool_path) for tool_path in all_tool_paths}
+                extra_volumes_str = ",".join(
+                    str(v) for v in create_docker_volumes(volumes) if v.path not in all_tool_dirs
+                )
+                volumes_str = f"{volumes_str},{extra_volumes_str}"
+            planemo_dest[f"{container_type}_volumes"] = volumes_str
+            break
+
+        JOB_CONFIG_LOCAL["execution"]["environments"]["planemo_dest"] = planemo_dest
+        with open(job_config_file, "w") as job_config_fh:
+            ordered_dump(JOB_CONFIG_LOCAL, job_config_fh)
     kwds["job_config_file"] = job_config_file
 
 
@@ -1433,6 +1440,8 @@ def _handle_container_resolution(ctx, kwds, galaxy_properties):
         involucro_context = build_involucro_context(ctx, **kwds)
         galaxy_properties["involucro_auto_init"] = "False"  # Use planemo's
         galaxy_properties["involucro_path"] = involucro_context.involucro_bin
+    if kwds.get("container_resolvers_config_file"):
+        galaxy_properties["container_resolvers_config_file"] = kwds.get("container_resolvers_config_file")
 
 
 def _handle_job_metrics(config_directory, kwds):

planemo/options.py

@@ -4,7 +4,10 @@
 import os
 
 import click
-from galaxy.tool_util.deps import docker_util
+from galaxy.tool_util.deps import (
+    docker_util,
+    singularity_util,
+)
 from galaxy.tool_util.verify.interactor import DEFAULT_TOOL_TEST_WAIT
 
 from .config import planemo_option
@@ -426,12 +429,20 @@ def tool_data_path_option():
 
 
 def mulled_containers_option():
-    return planemo_option(
-        "mulled_containers",
-        "--mulled_containers",
-        "--biocontainers",
-        is_flag=True,
-        help="Test tools against mulled containers (forces --docker). Disables conda resolution unless any conda option has been set explicitly.",
+    return _compose(
+        planemo_option(
+            "mulled_containers",
+            "--mulled_containers",
+            "--biocontainers",
+            is_flag=True,
+            help="Test tools against mulled containers (requires --docker/--singularity, if none of these are given --docker is used automatically). Disables conda resolution unless any conda option has been set explicitly.",
+        ),
+        planemo_option(
+            "--container_resolvers_config_file",
+            type=click.Path(exists=True, file_okay=True, dir_okay=False, resolve_path=True),
+            help="Container resolver config file",
+            default=None,
+        ),
     )
 
 
@@ -471,13 +482,23 @@ def docker_extra_volume_option():
         readable=True,
         resolve_path=True,
     )
-    return planemo_option(
-        "--docker_extra_volume",
-        type=arg_type,
-        default=None,
-        use_global_config=True,
-        multiple=True,
-        help=("Extra path to mount if --engine docker or `--biocontainers` or `--docker`."),
+    return _compose(
+        planemo_option(
+            "--docker_extra_volume",
+            type=arg_type,
+            default=None,
+            use_global_config=True,
+            multiple=True,
+            help=("Extra path to mount if --engine docker or `--biocontainers` or `--docker`."),
+        ),
+        planemo_option(
+            "--singularity_extra_volume",
+            type=arg_type,
+            default=None,
+            use_global_config=True,
+            multiple=True,
+            help=("Extra path to mount if `--biocontainers` and `--singularity`."),
+        ),
     )
 
 
@@ -945,27 +966,52 @@ def no_cleanup_option():
 
 
 def docker_enable_option():
-    return planemo_option("--docker/--no_docker", default=False, help=("Run Galaxy tools in Docker if enabled."))
+    return _compose(
+        planemo_option("--docker/--no_docker", default=False, help=("Run Galaxy tools in Docker if enabled.")),
+        planemo_option(
+            "--singularity/--no_singularity", default=False, help=("Run Galaxy tools in Singularity if enabled.")
+        ),
+    )
 
 
 def docker_cmd_option():
-    return planemo_option(
-        "--docker_cmd",
-        default=docker_util.DEFAULT_DOCKER_COMMAND,
-        help="Command used to launch docker (defaults to docker).",
+    return _compose(
+        planemo_option(
+            "--docker_cmd",
+            default=docker_util.DEFAULT_DOCKER_COMMAND,
+            help=f"Command used to launch docker (defaults to {docker_util.DEFAULT_DOCKER_COMMAND}).",
+        ),
+        planemo_option(
+            "--singularity_cmd",
+            default=singularity_util.DEFAULT_SINGULARITY_COMMAND,
+            help=f"Command used to launch singularity (defaults to {singularity_util.DEFAULT_SINGULARITY_COMMAND}).",
+        ),
     )
 
 
 def docker_sudo_option():
-    return planemo_option("--docker_sudo/--no_docker_sudo", is_flag=True, help="Flag to use sudo when running docker.")
+    return _compose(
+        planemo_option("--docker_sudo/--no_docker_sudo", is_flag=True, help="Flag to use sudo when running docker."),
+        planemo_option(
+            "--singularity_sudo/--no_singularity_sudo", is_flag=True, help="Flag to use sudo when running singularity."
+        ),
+    )
 
 
 def docker_sudo_cmd_option():
-    return planemo_option(
-        "--docker_sudo_cmd",
-        help="sudo command to use when --docker_sudo is enabled " + "(defaults to sudo).",
-        default=docker_util.DEFAULT_SUDO_COMMAND,
-        use_global_config=True,
+    return _compose(
+        planemo_option(
+            "--docker_sudo_cmd",
+            help=f"sudo command to use when --docker_sudo is enabled (defaults to {docker_util.DEFAULT_SUDO_COMMAND}).",
+            default=docker_util.DEFAULT_SUDO_COMMAND,
+            use_global_config=True,
+        ),
+        planemo_option(
+            "--singularity_sudo_cmd",
+            help=f"sudo command to use when --singularity_sudo is enabled (defaults to {singularity_util.DEFAULT_SUDO_COMMAND}).",
+            default=docker_util.DEFAULT_SUDO_COMMAND,
+            use_global_config=True,
+        ),
     )