Use Docker containers in the RTW (#3399)

Co-authored-by: Alistair Sellar <[email protected]>
Co-authored-by: Emma Hogan <[email protected]>

esmvaltool/utils/recipe_test_workflow/doc/source/user_guide/workflow.rst

@@ -18,13 +18,18 @@ The |RTW| performs the following steps:
   :Details:
      Runs once at the start of the workflow
 
-``clone_latest_esmval``
+``get_esmval``
   :Description:
-     Checks out the latest versions of |ESMValTool| and |ESMValCore| from GitHub
+     Either clones the latest versions of |ESMValTool| and |ESMValCore| from GitHub, 
+     or gets the latest container image from DockerHub and converts to a singularity
+     image, depending on ``SITE``.
   :Runs on:
-     Localhost
+     Localhost (if cloning), or ``COMPUTE`` (if getting container), which
+     depends on the ``SITE``; on JASMIN, the ``get_esmval`` jobs will run on
+     LOTUS
   :Executes:
-     The ``clone_latest_esmval.sh`` script from the |Rose| app
+     The ``clone_latest_esmval.sh`` script (if cloning), or a ``singularity build``
+     command (if getting container) from the |Rose| app
   :Details:
      Runs at the start of each cycle
 

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py

@@ -30,8 +30,7 @@ def main():
 
 
 def get_config_values_from_task_env():
-    """Return the configuration values defined in the environment for the
-    ``configure`` task."""
+    """Get configuration values from environment for configure task."""
     # Note that 'auxiliary_data_dir' and 'download_dir' are set to empty
     # values and cannot currently be configured. 'auxiliary_data_dir' is
     # used by some recipes to look for additional datasets, so may need
@@ -82,8 +81,7 @@ def get_config_values_from_task_env():
 
 
 def write_yaml(file_path, contents):
-    """Write the contents specified by ``contents`` to the YAML file specified
-    by ``file_path``.
+    """Write ``contents`` to the YAML file ``file_path``.
 
     Parameters
     ----------
@@ -92,7 +90,7 @@ def write_yaml(file_path, contents):
     contents: dictionary
         The contents to write to the YAML file.
     """
-    with open(file_path, "w") as file_handle:
+    with open(file_path, "w", encoding="utf-8") as file_handle:
         yaml.dump(contents, file_handle, default_flow_style=False)
 
 

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/get_esmval/opt/rose-app-jasmin.conf

@@ -0,0 +1,10 @@
+[command]
+default=singularity build ${CONTAINER_DIR}/${CONTAINER_FILE} ${DOCKER_SOURCE}; rtw-env pip install imagehash --user
+
+[env]
+CONTAINER_DIR=${ROSE_DATAC}/container
+CONTAINER_FILE=esmvaltool.sif
+DOCKER_SOURCE=docker://esmvalgroup/esmvaltool:${ENV_NAME}
+
+[file:${CONTAINER_DIR}]
+mode=mkdir

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/flow.cylc

@@ -11,33 +11,34 @@
    initial cycle point = 1
    final cycle point = 10
     [[graph]]
-        R1 = """install_env_file => configure & clone_latest_esmval
-                configure & clone_latest_esmval => process<medium> => compare<medium>
-                configure & clone_latest_esmval => process<fast> => compare<fast>"""
+
+        R1 = """install_env_file => get_esmval => configure
+                configure => process<medium> => compare<medium>
+                configure => process<fast> => compare<fast>"""
 
 [runtime]
     [[root]]
         script = rose task-run
         env-script = "eval $(rose task-env)"
         [[[environment]]]
-            MODULE_NAME = {{ MODULE_NAME }}
+            ENV_NAME = {{ ENV_NAME }}
             USER_CONFIG_DIR = ${CYLC_WORKFLOW_RUN_DIR}/etc
             USER_CONFIG_PATH = ${USER_CONFIG_DIR}/config-user.yml
             OUTPUT_DIR = ${CYLC_WORKFLOW_SHARE_DIR}/cycle/${CYLC_TASK_CYCLE_POINT}
-            ESMVALCORE_DIR = ${CYLC_WORKFLOW_RUN_DIR}/lib/ESMValCore
-            ESMVALTOOL_DIR = ${CYLC_WORKFLOW_RUN_DIR}/lib/ESMValTool
-            PYTHONPATH_PREPEND = ${ESMVALCORE_DIR}:${ESMVALTOOL_DIR}
+            ESMVALCORE_DIR = ""
+            ESMVALTOOL_DIR = ""
+            PYTHONPATH_PREPEND = ""
 
     [[install_env_file]]
         platform = localhost
         [[[environment]]]
             SITE = {{ SITE }}
 
-    [[clone_latest_esmval]]
-        platform = localhost
+    [[get_esmval]]
+        [[[environment]]]
+            ROSE_APP_OPT_CONF_KEYS = {{ SITE }}
 
     [[configure]]
-        platform = localhost
         pre-script = "mkdir -p ${USER_CONFIG_DIR}"
         [[[environment]]]
             ROSE_TASK_APP = configure

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/meta/rose-meta.conf

@@ -79,6 +79,15 @@ help=To override the default value for a specific site, define the override
     =directory.
 type=quoted
 
+[template variables=ENV_NAME]
+compulsory=false
+description=The name of the environment (module or container label) to load for
+           =the RTW.
+help=If required, this value must be set in a site-specific configuration file
+    =located in the 'opt/' directory. Otherwise, set the value equal to an
+    =empty string.
+type=quoted
+
 [template variables=MAX_PARALLEL_TASKS]
 compulsory=true
 description=The number of tasks (within a single call to ESMValTool) to run in
@@ -88,15 +97,6 @@ help=To override the default value for a specific site, define the override
     =directory.
 type=integer
 
-[template variables=MODULE_NAME]
-compulsory=false
-description=The name of the module to load to setup the environment for the
-           =RTW.
-help=If required, this value must be set in a site-specific configuration file
-    =located in the 'opt/' directory. Otherwise, set the value equal to an
-    =empty string.
-type=quoted
-
 [template variables=ROOTPATH_ANA4MIPS]
 compulsory=false
 description=The root path to the input ana4mips data.

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/opt/rose-suite-jasmin.conf

@@ -1,8 +1,8 @@
 [template variables]
 DRS_CMIP5="BADC"
 DRS_CMIP6="BADC"
-KGO_ROOT_PATH="/gws/nopw/j04/esmeval/KGO_v2.7.0/"
-MODULE_NAME="esmvaltool/2.7"
+ENV_NAME="latest"
+KGO_ROOT_PATH="/gws/nopw/j04/esmeval/KGO"
 ROOTPATH_CMIP5="/badc/cmip5/data/cmip5/output1"
 ROOTPATH_CMIP6="/badc/cmip6/data/CMIP6"
 ROOTPATH_OBS="/gws/nopw/j04/esmeval/obsdata-v2"

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/opt/rose-suite-metoffice.conf

@@ -1,8 +1,8 @@
 [template variables]
 DRS_CMIP5="BADC"
 DRS_CMIP6="BADC"
+ENV_NAME="scitools/community/esmvaltool/2.9.0-2"
 KGO_ROOT_PATH="/data/users/esmval/KGO_v2.9.0/"
-MODULE_NAME="scitools/community/esmvaltool/2.9.0-2"
 ROOTPATH_CMIP5="/project/champ/data/cmip5/output1"
 ROOTPATH_CMIP6="/project/champ/data/CMIP6"
 ROOTPATH_OBS="/data/users/esmval/ESMValTool/temporary/obs/"

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/rose-suite.conf

@@ -8,9 +8,9 @@ DRS_NATIVE6="default"
 DRS_OBS="default"
 DRS_OBS4MIPS="default"
 DRS_OBS6="default"
+ENV_NAME=""
 KGO_ROOT_PATH=""
 MAX_PARALLEL_TASKS=4
-MODULE_NAME=""
 ROOTPATH_ANA4MIPS=""
 ROOTPATH_CMIP3=""
 ROOTPATH_CMIP5=""

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/site/jasmin-env

@@ -1,32 +1,16 @@
-#!/bin/bash -l
+#!/bin/bash
 #
 # USAGE jasmin-env COMMAND
 #
 # ENVIRONMENT
-#   MODULE_NAME         The name of the module to load
-#   PYTHONPATH_PREPEND  The path to prepend to PYTHONPATH after loading the
-#                       module
+#   ENV_NAME            The name of the container being used
 #   QUIET_MODE          Don't print confirmation messages
+#   PYTHONPATH_PREPEND  The path to prepend to PYTHONPATH
 #
 # OPTIONS
 #   COMMAND             The command to execute with options
 set -eu
 
-module_count(){
-    module list -t 2>&1 | wc -l
-}
-
-safe_load(){
-    PRE_LOAD_COUNT=$(module_count)
-
-    module load "${1}"
-    # Check module count to determine whether module load was successful.
-    if (( PRE_LOAD_COUNT == $(module_count) )); then
-        echo "[ERROR] Failed to load: ${1}"
-        exit 1
-    fi
-}
-
 # Must be run before importing numpy, see
 # https://www-avd/sci/dask_best_practice/numpy-threads.html.
 export OMP_NUM_THREADS=1
@@ -38,7 +22,27 @@ export NUMEXPR_NUM_THREADS=1
 # Ensure '~/.local' isn't added to 'sys.path'.
 export PYTHONNOUSERSITE=True
 
-safe_load "${MODULE_NAME}"
+WORKFLOW_RUN_BIN_DIR="${CYLC_WORKFLOW_RUN_DIR}/bin"
+WORKFLOW_SHARE_BIN_DIR="${CYLC_WORKFLOW_SHARE_DIR}/cycle/bin"
+ROSE_APP_BIN_DIR="${CYLC_WORKFLOW_RUN_DIR}/app/${ROSE_TASK_APP:-$CYLC_TASK_NAME}/bin"
+CONTAINER=${ROSE_DATAC}/container/esmvaltool.sif
+
+# Bind paths for container. Where symbolic links are used in file paths (e.g.
+# under /badc/cmip6 and /home/users) need to bind the root dirs of both the
+# source and target files or directories.
+export SINGULARITY_BIND="/badc,/datacentre,/home/users,/work,/gws"
+
+# Suppress an ESMValTool "file not found" warning 
+export SINGULARITYENV_PROJ_DATA="/opt/conda/envs/esmvaltool/share/proj"
+
+# Provide mkfile needed to build esmfpy package
+export SINGULARITYENV_ESMFMKFILE="/opt/conda/envs/esmvaltool/lib/esmf.mk"
+
+# Ensure that `singularity exec` finds the right version of python
+export SINGULARITYENV_PREPEND_PATH="/opt/conda/envs/esmvaltool/bin"
+
+# Include Rose/Cylc workflow directories in container PATH
+export SINGULARITYENV_APPEND_PATH="${WORKFLOW_RUN_BIN_DIR}:${WORKFLOW_SHARE_BIN_DIR}:${ROSE_APP_BIN_DIR}"
 
 # If PYTHONPATH_PREPEND has been set, prepend it to PYTHONPATH to extend the
 # Python environment.
@@ -47,9 +51,9 @@ if [[ ! -z ${PYTHONPATH_PREPEND:-} ]]; then
     export PYTHONPATH=${PYTHONPATH_PREPEND}:${PYTHONPATH:-}
 fi
 
+
 if [[ -z ${QUIET_MODE:-} ]]; then
-    echo "[OK] Modules loaded."
+    echo "[INFO] Using the ${ENV_NAME} container"
 fi
 
-command="/usr/bin/time -v -o ${CYLC_TASK_LOG_ROOT}.time $@"
-exec ${command}
+/usr/bin/time -v -o "${CYLC_TASK_LOG_ROOT}.time" singularity -q exec "${CONTAINER}" "$@"

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/site/jasmin.cylc

@@ -1,17 +1,36 @@
 #!jinja2
 [runtime]
+    [[root]]
+        [[[environment]]]
+            # Warning: fragile assumption of location of esmvaltool code, see #3437
+            ESMVALTOOL_DIR = /opt/conda/envs/esmvaltool/lib/python3.11/site-packages
+            # Pick up local installation of imagehash. Slightly fragile: see #3436
+            PYTHONPATH_PREPEND = ${HOME}/.local/lib/python3.11/site-packages
+
     [[COMPUTE]]
         platform = lotus
         [[[directives]]]
             --wckey = RTW
             --ntasks = 1
 
-    [[process<fast>]]
+    [[get_esmval]]
+        inherit = None, COMPUTE
+        execution time limit = PT30M
+        [[[directives]]]
+            --mem = 2G
+
+    [[configure]]
+        inherit = None, COMPUTE
         execution time limit = PT2M
         [[[directives]]]
             --mem = 2G
 
+    [[process<fast>]]
+        execution time limit = PT5M
+        [[[directives]]]
+            --mem = 2G
+
     [[process<medium>]]
-        execution time limit = PT6M
+        execution time limit = PT10M
         [[[directives]]]
             --mem = 2G

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/site/metoffice-env

@@ -3,7 +3,7 @@
 # USAGE metoffice-env COMMAND
 #
 # ENVIRONMENT
-#   MODULE_NAME         The name of the module to load
+#   ENV_NAME            The name of the module to load
 #   PYTHONPATH_PREPEND  The path to prepend to PYTHONPATH after loading the
 #                       module
 #   QUIET_MODE          Don't print confirmation messages
@@ -38,7 +38,7 @@ export NUMEXPR_NUM_THREADS=1
 # Ensure '~/.local' isn't added to 'sys.path'.
 export PYTHONNOUSERSITE=True
 
-safe_load "${MODULE_NAME}"
+safe_load "${ENV_NAME}"
 
 # If PYTHONPATH_PREPEND has been set, prepend it to PYTHONPATH to extend the
 # Python environment.

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/site/metoffice.cylc

@@ -1,11 +1,26 @@
 #!jinja2
 [runtime]
+    [[root]]
+        [[[environment]]]
+            ESMVALCORE_DIR = ${CYLC_WORKFLOW_RUN_DIR}/lib/ESMValCore
+            ESMVALTOOL_DIR = ${CYLC_WORKFLOW_RUN_DIR}/lib/ESMValTool
+            PYTHONPATH_PREPEND = ${ESMVALCORE_DIR}:${ESMVALTOOL_DIR}
+
     [[COMPUTE]]
         platform = spice
         [[[directives]]]
             --wckey = RTW
             --ntasks = {{ MAX_PARALLEL_TASKS }}
 
+    [[configure]]
+        platform = localhost
+
+    [[get_esmval]]
+        platform = localhost
+        execution time limit = PT30M
+        [[[directives]]]
+            --mem = 2G
+
     [[process<fast>]]
         execution time limit = PT2M
         [[[directives]]]