CMORiser for ANU Climate 2.0 Australian data (#3511)

Co-authored-by: Axel Lauer <[email protected]> Co-authored-by: Manuel Schlund <[email protected]> Co-authored-by: Bouwe Andela <[email protected]> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: valeriupredoi <[email protected]> Co-authored-by: Valeriu Predoi <[email protected]> Co-authored-by: Björn Brötz <[email protected]> Co-authored-by: rbeucher <[email protected]> Co-authored-by: Lisa Bock <[email protected]> Co-authored-by: Klaus Zimmermann <[email protected]>
ESMValGroup · May 9, 2024 · cb582bd · cb582bd
1 parent 85e24b5
commit cb582bd
Show file tree

Hide file tree

Showing 6 changed files with 192 additions and 0 deletions.
diff --git a/doc/sphinx/source/input.rst b/doc/sphinx/source/input.rst
@@ -234,6 +234,8 @@ A list of the datasets for which a CMORizers is available is provided in the fol
 +==============================+======================================================================================================+======+=================+
 | AGCD                         | pr (Amon)                                                                                            |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
+| ANU Climate                  | pr, tas, tasmin, tasmax (Amon)                                                                       |   3  | Python          |
++------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | APHRO-MA                     | pr, tas (day), pr, tas (Amon)                                                                        |   3  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | AURA-TES                     | tro3 (Amon)                                                                                          |   3  | NCL             |

diff --git a/esmvaltool/cmorizers/data/cmor_config/ANUClimate.yml b/esmvaltool/cmorizers/data/cmor_config/ANUClimate.yml
@@ -0,0 +1,32 @@
+---
+filename: 'ANUClimate_{version}_{raw}_{freq}_.*.nc' #yyyymm
+
+attributes:
+  project_id: OBS6
+  dataset_id: ANUClimate
+  version: 'v2-0'
+  tier: 3
+  modeling_realm: reanaly
+  # resolution: '0.01 degree'
+  source: 'https://dx.doi.org/10.25914/60a10aa56dd1b'
+  reference: 'anuclimate2'
+  comment: 'hosted on NCI'
+
+variables:
+  pr:
+    mip: Amon
+    freq: monthly  # daily available
+    raw: rain
+
+  tasmax:
+    mip: Amon
+    freq: monthly
+    raw: tmax
+  tasmin:
+    mip: Amon
+    freq: monthly
+    raw: tmin
+  tas:
+    mip: Amon
+    freq: monthly
+    raw: tavg
diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml
@@ -19,6 +19,17 @@ datasets:
       Data from NCI (National Computing Infrastructure Australia https://nci.org.au/), 
       requires an NCI account and access to Gadi(Supercomputer in Canberra) and the project found in catalogue record.
       Access can be requested through NCI. NCI is an ESGF node (https://esgf.nci.org.au/projects/esgf-nci/)
+  
+  ANUClimate:
+    tier: 3
+    source: "https://dx.doi.org/10.25914/60a10aa56dd1b"
+    last_access: 2023-11-21
+    info: |
+      Data from NCI project requiring an NCI account and access to GADI
+
+      ANUClimate 2.0 consists of gridded daily and monthly climate variables across the terrestrial landmass of Australia
+      from at least 1970 to the present. Rainfall grids are generated from 1900 to the present. The underpinning spatial 
+      models have been developed at the Fenner School of Environment and Society of the Australian National University.
 
   APHRO-MA:
     tier: 3

diff --git a/esmvaltool/cmorizers/data/formatters/datasets/anuclimate.py b/esmvaltool/cmorizers/data/formatters/datasets/anuclimate.py
@@ -0,0 +1,125 @@
+"""ESMValTool CMORizer for ANU Climate data.
+
+Tier
+    Tier 3: restricted dataset.
+
+Source
+    https://dx.doi.org/10.25914/60a10aa56dd1b
+
+Last access
+    20231121
+
+Download and processing instructions
+    Data from NCI project requiring an NCI account and access to GADI
+    Processing is done on GADI
+
+"""
+import logging
+import os
+import re
+import calendar
+
+import iris
+
+from esmvaltool.cmorizers.data import utilities as utils
+
+logger = logging.getLogger(__name__)
+
+
+def _get_filepaths(in_dir, basename):
+    """Find correct name of file (extend basename with timestamp)."""
+    regex = re.compile(basename)
+    return_files = []
+    # Search sub folders of raw data directory
+    for root, _dir, files in os.walk(in_dir, followlinks=True):
+
+        for filename in files:
+            if regex.match(filename):
+
+                return_files.append(os.path.join(root, filename))
+
+    return return_files
+
+
+def fix_data_var(cube, var):
+    """Convert units in cube for the variable."""
+    # get month, year from cube
+    tcoord = cube.coord('time')
+    tdate = tcoord.units.num2date(tcoord.points[0])
+    no_ofdays = calendar.monthrange(tdate.year, tdate.month)[1]
+
+    if var == 'pr':
+
+        cube = cube / (no_ofdays * 86400)  # days in month
+        cube.units = 'kg m-2 s-1'
+
+    elif var in ['tas', 'tasmin', 'tasmax']:  # other variables in v1
+        cube = cube + 273.15
+        cube.units = 'K'
+        utils.add_height2m(cube)
+
+    else:
+        logger.info("Variable %s not converted", var)
+
+    return cube, tdate.year
+
+
+def _extract_variable(cmor_info, attrs, filepaths, out_dir):
+    """Extract variable."""
+    var = cmor_info.short_name
+    logger.info("Var is %s", var)
+    cbls_2 = iris.cube.CubeList()
+    cbls_1 = iris.cube.CubeList()
+    for filepath in filepaths:
+        cubes = iris.load(filepath)
+
+        cube, year = fix_data_var(cubes[0], var)
+
+        utils.fix_var_metadata(cube, cmor_info)
+
+        utils.set_global_atts(cube, attrs)
+
+        if year < 2000:  # split for cube save
+            cbls_1.append(cube)
+        else:
+            cbls_2.append(cube)
+
+    for cbls in [cbls_1, cbls_2]:
+        iris.util.equalise_attributes(cbls)
+        cubesave = cbls.concatenate_cube()
+        utils.fix_coords(cubesave)
+
+        logger.info("Saving file")
+        utils.save_variable(cubesave,
+                            var,
+                            out_dir,
+                            attrs,
+                            unlimited_dimensions=['time'])
+
+
+def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date):
+    """Cmorization func call."""
+    glob_attrs = cfg['attributes']
+    cmor_table = cfg['cmor_table']
+
+    ver = cfg['attributes']['version']
+    logger.info(cfg, cfg_user)
+
+    # Run the cmorization, multiple variables
+    for (var, var_info) in cfg['variables'].items():
+
+        glob_attrs['mip'] = var_info['mip']
+
+        raw_filename = cfg['filename'].format(version=ver,
+                                              raw=var_info['raw'],
+                                              freq=var_info['freq'])
+        filepaths = _get_filepaths(in_dir, raw_filename)
+
+        if len(filepaths) == 0:
+            logger.info("no files for %s pattern: %s", var, raw_filename)
+            logger.info("directory: %s", in_dir)
+        else:
+            logger.info("Found files, count %s", len(filepaths))
+
+            cmor_info = cmor_table.get_variable(var_info['mip'], var)
+            _extract_variable(cmor_info, glob_attrs, filepaths, out_dir)
diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml
@@ -906,6 +906,17 @@ diagnostics:
 
   ### TIER 3 ##################################################################
 
+  ANUClimate:
+    description: ANUClimate check
+    variables:
+      pr:
+      tasmax:
+      tasmin:
+      tas:
+    additional_datasets:
+      - {dataset: ANUClimate, project: OBS6, mip: Amon, tier: 3, type: reanaly}
+    scripts: null
+
   APHRO-MA:
     description: APHRO-MA check
     variables:

diff --git a/esmvaltool/references/anuclimate2.bibtex b/esmvaltool/references/anuclimate2.bibtex
@@ -0,0 +1,11 @@
+@misc{https://doi.org/10.25914/60a10aa56dd1b,
+  doi = {10.25914/60A10AA56DD1B},
+  url = {https://pid.nci.org.au/doi/f2576_7854_4065_1457},
+  author = {Hutchinson,  Michael and Xu,  Tingbao and Kesteven,  Jennifer and Marang,  Ian and Evans,  Bradley},
+  keywords = {Climatology (excl. Climate Change Processes)},
+  language = {en},
+  title = {ANUClimate 2.0},
+  publisher = {NCI Australia},
+  year = {2021},
+  copyright = {Creative Commons Attribution Share Alike 4.0 International}
+}