Upgrade the RTW to work with ESMValTool v2.8.0

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
       - id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
   - repo: https://github.com/adrienverge/yamllint
-    rev: 'v1.26.0'
+    rev: 'v1.30.0'
     hooks:
       - id: yamllint
   - repo: local  # nclcodestyle is installed alongside ESMValTool
@@ -39,7 +39,7 @@ repos:
     hooks:
       - id: codespell
   - repo: https://github.com/PyCQA/isort
-    rev: '5.7.0'
+    rev: '5.12.0'
     hooks:
       - id: isort
   - repo: https://github.com/pre-commit/mirrors-yapf

esmvaltool/utils/recipe_test_workflow/doc/source/about.rst

@@ -5,7 +5,7 @@ About
 .. include:: common.txt
 
 The Recipe Test Workflow (|RTW|) is a workflow that is used to regularly run
-recipes so issues can be discovered during the development process sooner 
+recipes so issues can be discovered during the development process sooner
 rather than later.
 
 |Cylc| v8 and |Rose| v2 are used as the workflow engine and application

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/configure/bin/configure.py

@@ -1,58 +1,84 @@
 #!/usr/bin/env python
 """Generate the required user configuration file for ESMValTool."""
 import os
-import warnings
-
-with warnings.catch_warnings():
-    warnings.filterwarnings(
-        "ignore",
-        message=(
-            "\n  Thank you for trying out the new ESMValCore API.\n  Note "
-            "that this API is experimental and may be subject to change.\n  "
-            "More info: https://github.com/ESMValGroup/ESMValCore/issues/498"),
-        category=UserWarning,
-        module="esmvalcore.experimental._warnings",
-    )
-    import esmvalcore.experimental as esmvaltool
+import pprint
+
 import yaml
+from esmvalcore.config._config_object import Config
 
-USER_CONFIG_PATH = os.environ["USER_CONFIG_PATH"]
+import esmvaltool
 
 
 def main():
-    """Write the updated configuration values to the file defined by
-    ``USER_CONFIG_PATH``."""
-    # Get the default configuration values from ESMValTool. A dictionary is
-    # needed here to avoid the config object from converting paths to PosixPath
-    # objects, which causes issues when writing the YAML file.
-    config_values = dict(esmvaltool.CFG)
+    """Write the required user configuration file for ESMValTool.
+
+    The default configuration values from the latest version of
+    ESMValTool are updated with the configuration values defined in the
+    environment for the ``configure`` task and the result is written to
+    the file defined by ``USER_CONFIG_PATH`` in the ``flow.cylc`` file.
+    """
+    # Get the default configuration values from the latest version of
+    # ESMValTool.
+    config_values = get_default_config_values_from_latest_ESMValTool()
 
     # Get the configuration values defined in the environment for the
-    # ``configure`` task.
+    # 'configure' task.
     config_values_from_task_env = get_config_values_from_task_env()
 
     # Update the default configuration values.
     config_values.update(config_values_from_task_env)
+    user_config_path = os.environ["USER_CONFIG_PATH"]
+    config_values["config_file"] = user_config_path
 
     # Write the updated configuration values to the file defined by
-    # 'USER_CONFIG_PATH'.
-    write_yaml(USER_CONFIG_PATH, config_values)
+    # 'user_config_path'.
+    print(f"Writing the user configuration file to '{user_config_path}' with "
+          "values: ")
+    pprint.PrettyPrinter().pprint(config_values)
+    write_yaml(user_config_path, config_values)
+
+
+def get_default_config_values_from_latest_ESMValTool():
+    """Return the default configuration values from the latest version of
+    ESMValTool."""
+    latest_esmvaltool_dir = os.environ["ESMVALTOOL_DIR"]
+    default_config_file = os.path.join(latest_esmvaltool_dir,
+                                       "config-user-example.yml")
+    # A dictionary is needed here to avoid the config object from
+    # converting paths to 'PosixPath' objects, which causes issues when
+    # writing the YAML file.
+    config_values = dict(Config._load_user_config(default_config_file))
+
+    # Update the type of the value of 'config_developer_file' from a
+    # 'PosixPath' object to a string, to avoid issues when writing the
+    # YAML file (all other paths will be overwritten).
+    config_developer_file = str(config_values["config_developer_file"])
+    config_values["config_developer_file"] = config_developer_file
+
+    print("Default configuration values from the latest version of ESMValTool "
+          f"{esmvaltool.__version__} (from '{default_config_file}'): ")
+    pprint.PrettyPrinter().pprint(config_values)
+
+    return config_values
 
 
 def get_config_values_from_task_env():
     """Return the configuration values defined in the environment for the
     ``configure`` task."""
-    # Note that 'auxiliary_data_dir', 'download_dir' and
-    # 'extra_facets_dir' are set to empty values and cannot currently be
-    # configured. However, 'download_dir' is used only when using the
-    # automatic download feature via ESMValTool (which we do not intend
-    # to use here) and 'extra_facets_dir' is not available in the
-    # default configuration file provided by ESMValTool v2.6.0.
-    # 'auxiliary_data_dir' is used by some recipes to look for
-    # additional datasets, so may need to be configured in the future.
+    # Note that 'auxiliary_data_dir' and 'download_dir' are set to empty
+    # values and cannot currently be configured. 'auxiliary_data_dir' is
+    # used by some recipes to look for additional datasets, so may need
+    # to be configured in the future. 'download_dir' is used only when
+    # using the automatic download feature via ESMValTool, which is not
+    # the intention here, so should not be configurable.
+    #
+    # In addition, 'check_level' and 'extra_facets_dir' are added to the
+    # config object in ESMValTool after loading the user configuration
+    # file, but they need updating to avoid issues when writing the YAML
+    # file.
     config_values_from_task_env = {
         "auxiliary_data_dir": "",
-        "config_file": USER_CONFIG_PATH,
+        "check_level": "DEFAULT",
         "download_dir": "",
         "drs": {
             "ana4mips": os.environ["DRS_ANA4MIPS"],
@@ -68,6 +94,7 @@ def get_config_values_from_task_env():
         "extra_facets_dir": [],
         "max_parallel_tasks": int(os.environ["MAX_PARALLEL_TASKS"]),
         "output_dir": os.environ["OUTPUT_DIR"],
+        "remove_preproc_dir": False,
         "rootpath": {
             "ana4mips": os.environ["ROOTPATH_ANA4MIPS"],
             "CMIP3": os.environ["ROOTPATH_CMIP3"],
@@ -81,6 +108,9 @@ def get_config_values_from_task_env():
             "RAWOBS": os.environ["ROOTPATH_RAWOBS"],
         },
     }
+    print("The configuration values defined in the environment for the "
+          "'configure' task: ")
+    pprint.PrettyPrinter().pprint(config_values_from_task_env)
     return config_values_from_task_env
 
 

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/app/install_cold/bin/install_cold.sh

@@ -31,7 +31,3 @@ fi
 # (which is done via done via 'stderr').
 git clone -q -b "${BRANCH}" "${ESMVALTOOL_URL}" "${ESMVALTOOL_DIR}"
 git clone -q -b "${BRANCH}" "${ESMVALCORE_URL}" "${ESMVALCORE_DIR}"
-
-# Manually and temporarily install imagehash (it will be included in the
-# next ESMValTool community environment).
-rtw-env pip install imagehash -t "${CYLC_WORKFLOW_RUN_DIR}/lib/python"

esmvaltool/utils/recipe_test_workflow/recipe_test_workflow/opt/rose-suite-metoffice.conf

@@ -3,10 +3,10 @@ DRS_CMIP5="BADC"
 DRS_CMIP6="BADC"
 DRS_OBS="default"
 DRS_OBS4MIPS="default"
-KGO_ROOT_PATH="/data/users/esmval/KGO_v2.6.0/"
-MODULE_NAME="scitools/community/esmvaltool/2.6.0"
+KGO_ROOT_PATH="/data/users/esmval/KGO/"
+MODULE_NAME="scitools/community/esmvaltool/2.8.0"
 ROOTPATH_CMIP5="/project/champ/data/cmip5/output1"
 ROOTPATH_CMIP6="/project/champ/data/CMIP6"
-ROOTPATH_OBS="data/users/esmval/ESMValTool/temporary/obs/"
+ROOTPATH_OBS="/data/users/esmval/ESMValTool/temporary/obs/"
 ROOTPATH_OBS4MIPS="/data/users/esmval/ESMValTool/temporary/obs/"
 SITE="metoffice"