From 2a6be12f474ee611f91bc8f9fc4b7162afac54f1 Mon Sep 17 00:00:00 2001 From: Manuel Schlund <32543114+schlunma@users.noreply.github.com> Date: Mon, 27 Jan 2025 14:28:50 +0100 Subject: [PATCH 1/2] Remove portrait plot test recipe (#3871) --- .../testing/recipe_portrait_CMIP_fast.yml | 160 ------------------ 1 file changed, 160 deletions(-) delete mode 100644 esmvaltool/recipes/testing/recipe_portrait_CMIP_fast.yml diff --git a/esmvaltool/recipes/testing/recipe_portrait_CMIP_fast.yml b/esmvaltool/recipes/testing/recipe_portrait_CMIP_fast.yml deleted file mode 100644 index 002df5b4a8..0000000000 --- a/esmvaltool/recipes/testing/recipe_portrait_CMIP_fast.yml +++ /dev/null @@ -1,160 +0,0 @@ -# ESMValTool ---- -documentation: - title: Performance metrics plots. - description: > - Test recipe for the performance comparison of CMIP simulations to a reference dataset. - authors: - - lindenlaub_lukas - maintainer: - - lindenlaub_lukas - references: - - eyring21ipcc - - gleckler08jgr - -cmip5: &CMIP5 - project: CMIP5 - ensemble: r1i1p1 - -datasets: - # cmip5 - - {<<: *CMIP5, dataset: ACCESS1-0} - - {<<: *CMIP5, dataset: CESM1-BGC} - - {<<: *CMIP5, dataset: GFDL-ESM2M} - - {<<: *CMIP5, dataset: MIROC-ESM} - - {<<: *CMIP5, dataset: MRI-CGCM3} - # cmip6 - - {dataset: ACCESS-ESM1-5, institute: CSIRO} - - {dataset: CESM2, institute: NCAR} - - {dataset: GFDL-CM4, grid: gr1} - - {dataset: MIROC-ES2L, ensemble: r1i1p1f2} - - {dataset: MRI-ESM2-0} - - -preprocessors: - default: &default # common preprocessor settings - regrid: - target_grid: 3x3 - scheme: linear - distance_metric: - metric: weighted_rmse - climate_statistics: - operator: mean - period: month - mask_fillvalues: - threshold_fraction: 0.95 - multi_model_statistics: - span: overlap - statistics: - - operator: mean - - operator: percentile - percent: 50 - groupby: ['project'] - # exclude all possible reference datasets - exclude: [ - AIRS-2-1, - CERES-EBAF, - ERA-Interim, - ESACCI-AEROSOL, - ESACCI-CLOUD, - ESACCI-OZONE, - ESACCI-SOILMOISTURE, - ESACCI-SST, - GPCP-SG, - HadISST, - NCEP-NCAR-R1, - MODIS, - NIWA-BS, - PATMOS-x] - pp200: # only add/overwrite var specific settings - <<: *default - extract_levels: - levels: 20000 - scheme: linear - coordinate: air_pressure - pp500: # only add/overwrite var specific settings - <<: *default - extract_levels: - levels: 50000 - scheme: linear - coordinate: air_pressure - thr10: # only add/overwrite var specific settings - <<: *default - mask_fillvalues: - threshold_fraction: 0.10 - -var_default: &var_default - mip: Amon - project: CMIP6 - exp: historical - ensemble: r1i1p1f1 - preprocessor: default - grid: gn - start_year: 2000 - end_year: 2002 - split: Ref1 # first triangle - -diagnostics: - portrait_rmse: - themes: [aerosols, phys, clouds, atmDyn, chem, ghg] - realms: [atmos, land, atmosChem, ocean] - variables: - tas: &tas - <<: *var_default - short_name: tas - variable: tas - additional_datasets: - - {dataset: ERA-Interim, project: OBS6, type: reanaly, - version: 1, tier: 3, reference_for_metric: true} - tas_2: - <<: *tas - split: Ref2 - additional_datasets: - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - version: 1, tier: 2, reference_for_metric: true} - pr: - <<: *var_default - variable: pr - split: Ref1 - additional_datasets: - - {dataset: GPCP-SG, project: OBS, type: atmos, - version: 2.3, tier: 2, reference_for_metric: true} - rsut: - <<: *var_default - variable: rsut - start_year: 2001 - end_year: 2003 - additional_datasets: - - {dataset: CERES-EBAF, project: OBS, type: sat, version: Ed4.2, - tier: 2, reference_for_metric: true} - ua200: &ua200 - <<: *var_default - variable: ua200 - short_name: ua - preprocessor: pp200 - split: Ref1 - additional_datasets: - - {dataset: ERA-Interim, project: OBS6, type: reanaly, - version: 1, tier: 3, reference_for_metric: true} - ua200_2: - <<: *ua200 - split: Ref2 - additional_datasets: - - {dataset: NCEP-NCAR-R1, project: OBS6, type: reanaly, - version: 1, tier: 2, reference_for_metric: true} - - scripts: - portrait: - script: portrait_plot.py - x_by: dataset - y_by: variable # extra_facet - group_by: project - normalize: "centered_median" - default_split: Ref1 - nan_color: null - plot_kwargs: - vmin: -0.5 - vmax: +0.5 - cbar_kwargs: - label: Relative RMSE - extend: both From 983de783583a76614f545da5200f2d4a32e28aef Mon Sep 17 00:00:00 2001 From: chrisbillowsMO <152496175+chrisbillowsMO@users.noreply.github.com> Date: Mon, 27 Jan 2025 14:17:08 +0000 Subject: [PATCH 2/2] #3856: Use site-specific lists of recipes in the RTW Co-authored-by: Emma Hogan --- doc/sphinx/source/utils/RTW/add_a_recipe.rst | 177 ++++++++++-------- doc/sphinx/source/utils/RTW/common.txt | 14 +- .../source/utils/RTW/tested_recipes.rst | 19 +- .../source/utils/RTW/user_guide/workflow.rst | 11 ++ .../Jinja2Tests/file_exists.py | 37 ++++ .../utils/recipe_test_workflow/flow.cylc | 48 +++-- .../site/metoffice-recipes.jinja | 81 ++++++++ .../recipe_test_workflow/site/metoffice.cylc | 37 ---- 8 files changed, 276 insertions(+), 148 deletions(-) create mode 100644 esmvaltool/utils/recipe_test_workflow/Jinja2Tests/file_exists.py create mode 100644 esmvaltool/utils/recipe_test_workflow/site/metoffice-recipes.jinja diff --git a/doc/sphinx/source/utils/RTW/add_a_recipe.rst b/doc/sphinx/source/utils/RTW/add_a_recipe.rst index 6e495e1f1c..954b518a36 100644 --- a/doc/sphinx/source/utils/RTW/add_a_recipe.rst +++ b/doc/sphinx/source/utils/RTW/add_a_recipe.rst @@ -3,116 +3,141 @@ How to add a recipe to the |RTW| .. include:: common.txt -.. note:: - Before you follow these steps to add your recipe, you must be able to - successfully run the recipe with the latest version of ESMValTool on the - compute server you use at your site, as detailed by the ``platform`` option - in the ``[[COMPUTE]]`` section in the site-specific ``.cylc`` file in the - ``esmvaltool/utils/recipe_test_workflow/site/`` directory. +Overview +-------- -#. Open a `new ESMValTool issue`_ on GitHub, assign yourself to the issue, and - add the ``Recipe Test Workflow (RTW)`` label to the issue, see - `ESMValTool issue #3663`_ for an example. +To add a recipe to the |RTW| you will: -#. Create a branch. +* Run the recipe at your site +* Note the actual duration and memory usage +* Edit your site's recipe file +* Create the recipe's KGOs +* Request a review -#. Obtain the duration and memory usage of the recipe from the messages printed - to screen, or at the end of the ``run/main_log.txt`` file in the recipe - output directory after running your recipe on the compute cluster you use at - your site; these messages will look something like:: +The recipe will then run at your site whenever the |RTW| is run. - YYYY-MM-DD HH:MM:SS:sss UTC [12345] INFO Time for running the recipe was: 0:02:13.334742 - YYYY-MM-DD HH:MM:SS:sss UTC [12345] INFO Maximum memory used (estimate): 2.4 GB - [...] - YYYY-MM-DD HH:MM:SS:sss UTC [12345] INFO Run was successful +Preparation +----------- -#. Add the recipe to the ``[task parameters]`` section in the - ``esmvaltool/utils/recipe_test_workflow/flow.cylc`` file. +#. Open a `new ESMValTool issue`_ on GitHub. Assign yourself to the issue and + add the ``Recipe Test Workflow (RTW)`` label. `ESMValTool issue #3663`_ + provides an example. - .. hint:: - If the recipe takes less than 10 minutes to run then it should be added - to the ``fast`` option. Recipes that take longer than ten minutes should - be added to the ``medium`` option. +#. Create a branch. + +#. Run the recipe: + + * with the latest version of ESMValTool + * on the compute server you use at your site .. hint:: - The line added should follow the format of ``recipe_new_recipe, \``, - unless the line is the last one in the list, in which case the line added - should follow the format of ``recipe_new_recipe``. - -#. If the duration of the recipe is larger than the value specified by the - ``execution time limit`` option in the ``[[COMPUTE]]`` section in the - aforementioned site-specific ``.cylc`` file, and / or the memory usage of - the recipe is larger than the value specified by the ``--mem`` option in the - ``[[[directives]]]`` section in the ``[[COMPUTE]]`` section, add a section - (in alphabetical order) to this file as shown below (round the duration to - the nearest second):: - - [[process]] - # Actual: 0m31s, 2.5 GB on 2024-04-08. - execution time limit = PT2M - [[[directives]]] - --mem = 3G + Your compute server is defined in the + ``esmvaltool/utils/recipe_test_workflow/site/.cylc`` file as + follows:: + + [[COMPUTE]] + platform = + +#. Obtain the actual duration and memory usage of the recipe. This can be found + either in the message printed to screen, or at the end of the + ``run/main_log.txt`` file in the recipe output directory. The relevant lines will + look something like:: + + YYYY-MM-DD HH:MM:SS:sss UTC [12345] INFO Time for running the recipe was: 0:02:13.334742 + YYYY-MM-DD HH:MM:SS:sss UTC [12345] INFO Maximum memory used (estimate): 2.4 GB + [...] + YYYY-MM-DD HH:MM:SS:sss UTC [12345] INFO Run was successful + +Adding the recipe +----------------- + +#. Add the recipe in alphabetical order to either ``FAST_RECIPES`` or + ``MEDIUM_RECIPES`` in the ``-recipes.jinja`` file. It should look + something like:: + + { + 'recipe_path': 'recipe_a_fast_recipe', + 'actual': '2m13s, 2.4 GB on YYYY-MM-DD', + 'max_time': 'PT3M', + 'max_memory': '3G', + } + + .. important:: + Add the recipe to ``FAST_RECIPES`` if it takes *less* than 10 mins to + run at your site. Add the recipe to ``MEDIUM_RECIPES`` if it takes *more* + than 10 mins. .. hint:: - The ``fast`` key in the example task definition above - (``[[process]]``) should match name of the - option the recipe was added to in the ``[task parameters]`` section in - the ``esmvaltool/utils/recipe_test_workflow/flow.cylc`` file + The :ref:`site/site-recipes.jinja `. + file provides more information. .. hint:: - Set the ``execution time limit`` to 10-20% more than the actual duration. - For actual durations of up to ``1m45s``, set the ``execution time limit`` - to ``PT2M`` (2 minutes). + Set the ``max_time`` to 10-20% more than the actual duration. For actual + durations of up to ``1m45s``, set ``max_time`` to ``PT2M`` (2 minutes). .. hint:: Try not to regularly waste more than 500 MiB in memory usage. Typically, rounding the actual memory usage up to the nearest integer is acceptable. -#. Stop any running ``recipe_test_workflow`` workflows:: - - cylc stop recipe_test_workflow/* +Create the |KGOs| +----------------- #. Run the |RTW|, as detailed in the :ref:`quick_start_guide`; it is expected that the ``compare`` task will fail. -#. Update the Known Good Outputs (|KGOs|): + .. important:: + The ``compare`` task fails because the |KGOs| for the recipe do not yet + exist. This run of the |RTW| will generate the outputs that will be + used as |KGOs|. - * Recursively copy the recipe output directory (i.e. - ``recipe___