Adding HadEX3 CMORizer

esmvaltool/cmorizers/data/cmor_config/HadEX3.yml

@@ -0,0 +1,65 @@
+---
+filename: 'HadEX3_{raw}_{raw_frequency}.nc'
+
+# Common global attributes for CMORizer output
+attributes:
+  dataset_id: HadEX3
+  version: v3.0.4
+  tier: 2
+  modeling_realm: ground
+  project_id: OBS6
+  source: https://www.metoffice.gov.uk/hadobs/hadex3/download.html
+  reference: 'hadex3'
+  comment: 'ETCCDI (Expert Team on Climate Change Detection and Indices)'
+
+# Variables to cmorize
+variables:
+  txx_yr:
+    short_name: txx
+    raw: TXx
+    raw_units: degrees_C
+    raw_frequency: ANN
+    mip: yr
+  txx_mon:
+    short_name: txx
+    raw: TXx
+    raw_units: degrees_C
+    raw_frequency: MON
+    mip: Amon
+  tnn_yr:
+    short_name: tnn
+    raw: TNn
+    raw_units: degrees_C
+    raw_frequency: ANN
+    mip: yr
+  tnn_mon:
+    short_name: tnn
+    raw: TNn
+    raw_units: degrees_C
+    raw_frequency: MON
+    frequency: mon
+    mip: Amon
+  rx1day_yr:
+    short_name: rx1day
+    raw: Rx1day
+    raw_units: mm
+    raw_frequency: ANN
+    mip: yr
+  rx1day_mon:
+    short_name: rx1day
+    raw: Rx1day
+    raw_units: mm
+    raw_frequency: MON
+    mip: Amon
+  rx5day_yr:
+    short_name: rx5day
+    raw: Rx5day
+    raw_units: mm
+    raw_frequency: ANN
+    mip: yr
+  rx5day_mon:
+    short_name: rx5day
+    raw: Rx5day
+    raw_units: mm
+    raw_frequency: MON
+    mip: Amon

esmvaltool/cmorizers/data/datasets.yml

@@ -746,6 +746,16 @@ datasets:
         climatology
             [Source]/absolute_v5.nc
 
+  HadEX3:
+    tier: 2
+    source: https://www.metoffice.gov.uk/hadobs/hadex3/download_etccdi.html
+    last_access: 2025-02-05
+    info: |
+      Download and unzip "HadEX3_TXx_MON.nc.gz", "HadEX3_TXx_ANN.nc.gz", 
+      "HadEX3_TNn_MON.nc.gz", "HadEX3_TNn_ANN.nc.gz", 
+      "HadEX3_Rx1day_MON.nc.gz", "HadEX3_Rx1day_ANN.nc.gz",
+      "HadEX3_Rx5day_MON.nc.gz" and "HadEX3_Rx5day_ANN.nc.gz".
+
   HadISST:
     tier: 2
     source: http://www.metoffice.gov.uk/hadobs/hadisst/data/download.html

esmvaltool/cmorizers/data/downloaders/datasets/hadex3.py

@@ -0,0 +1,51 @@
+"""Script to download HadEX3 from its webpage."""
+import logging
+import os
+
+from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader
+from esmvaltool.cmorizers.data.utilities import unpack_files_in_folder
+
+logger = logging.getLogger(__name__)
+
+
+def download_dataset(config, dataset, dataset_info, start_date, end_date,
+                     overwrite):
+    """Download dataset.
+
+    Parameters
+    ----------
+    config : dict
+        ESMValTool's user configuration
+    dataset : str
+        Name of the dataset
+    dataset_info : dict
+         Dataset information from the datasets.yml file
+    start_date : datetime
+        Start of the interval to download
+    end_date : datetime
+        End of the interval to download
+    overwrite : bool
+        Overwrite already downloaded files
+    """
+    downloader = WGetDownloader(
+        config=config,
+        dataset=dataset,
+        dataset_info=dataset_info,
+        overwrite=overwrite,
+    )
+
+    os.makedirs(downloader.local_folder, exist_ok=True)
+
+    web_source = "https://www.metoffice.gov.uk/hadobs/hadex3/data/"
+
+    file_list = [
+        "HadEX3_TXx_MON.nc.gz", "HadEX3_TXx_ANN.nc.gz", "HadEX3_TNn_MON.nc.gz",
+        "HadEX3_TNn_ANN.nc.gz", "HadEX3_Rx1day_MON.nc.gz",
+        "HadEX3_Rx1day_ANN.nc.gz", "HadEX3_Rx5day_MON.nc.gz",
+        "HadEX3_Rx5day_ANN.nc.gz"
+    ]
+
+    for file in file_list:
+        downloader.download_file(f"{web_source}{file}", wget_options=[])
+
+    unpack_files_in_folder(downloader.local_folder)

esmvaltool/cmorizers/data/formatters/datasets/hadex3.py

@@ -0,0 +1,113 @@
+"""ESMValTool CMORizer for HadEX3 data.
+
+Tier
+    Tier 2: other freely-available dataset.
+
+Source
+    https://www.metoffice.gov.uk/hadobs/hadex3/download.html
+
+Last access
+    2025-02-05
+
+Download and processing instructions
+    Download the following files:
+        Annual.nc.gz and Monthly.nc.gz for TXx, TNn, Rx1day and Rx5day
+    No registration is required for downloading the data.
+
+When using the dataset in a paper, the following are citations to use:
+
+Dunn, R. J. H., et al. (2020), Development of an updated global land
+in-situ-based dataset of temperature and precipitation extremes: HadEX3 JGR-A
+
+Dunn, R. J. H., Donat, M. G., Alexander, L. V., al. (2014), Investigating
+uncertainties in global gridded datasets of climate extremes,
+Climate of the Past, 10, 2171-2199
+
+Donat, M. G., Alexander, L. V., .... Dunn, R. J. H., et al. (2013), Updated
+analyses of temperature and precipitation extreme indices since the beginning
+of the twentieth century: The HadEX2 dataset, J. Geophys. Res. Atmos., 118,
+2098-2118
+"""
+
+import logging
+import os
+
+import cftime
+import iris
+from cf_units import Unit
+from iris import NameConstraint
+
+from esmvaltool.cmorizers.data import utilities as utils
+
+logger = logging.getLogger(__name__)
+
+
+def _fix_time_coord(cube):
+    """Convert the time to the gregorian calendar."""
+    time_coord = cube.coord('time')
+    time_coord.guess_bounds()
+    new_unit = Unit('days since 1850-01-01 00:00:00', calendar='gregorian')
+    new_time_points = cftime.num2pydate(time_coord.points,
+                                        time_coord.units.origin,
+                                        time_coord.units.calendar)
+    time_points = cftime.date2num(new_time_points,
+                                  new_unit.origin,
+                                  calendar=new_unit.calendar)
+    new_time_bounds = cftime.num2pydate(time_coord.bounds,
+                                        time_coord.units.origin,
+                                        time_coord.units.calendar)
+    time_bounds = cftime.date2num(new_time_bounds,
+                                  new_unit.origin,
+                                  calendar=new_unit.calendar)
+
+    cube.coord('time').points = time_points
+    cube.coord('time').bounds = time_bounds
+    cube.coord('time').units = new_unit
+
+
+def _extract_variable(var, var_info, cmor_info, attrs, filepath, out_dir):
+    """Extract variable."""
+    raw_var = var_info.get('raw', var)
+    var = cmor_info.short_name
+    cube = iris.load_cube(filepath, NameConstraint(var_name=raw_var))
+    # Fix units
+    cube.units = var_info.get('raw_units', var)
+    cube.convert_units(cmor_info.units)
+
+    _fix_time_coord(cube)
+    utils.fix_var_metadata(cube, cmor_info)
+    utils.convert_timeunits(cube, 1950)
+    utils.fix_coords(cube)
+    utils.set_global_atts(cube, attrs)
+    utils.save_variable(cube,
+                        var,
+                        out_dir,
+                        attrs,
+                        unlimited_dimensions=['time'])
+
+
+def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date):
+    """Cmorization function call."""
+    glob_attrs = cfg['attributes']
+    cmor_table = cfg['cmor_table']
+    filename = cfg['filename']
+
+    # Run the cmorization
+    for (var, var_info) in cfg['variables'].items():
+        var_name = var_info['short_name']
+        var_mip = var_info['mip']
+        filepath = os.path.join(
+            in_dir,
+            filename.format(raw=var_info['raw'],
+                            raw_frequency=var_info['raw_frequency']))
+        if os.path.isfile(filepath):
+            logger.info(f"Found input file {filepath}")
+        else:
+            raise ValueError(f"Couldn't find {filepath}")
+        logger.info(f"CMORizing variable {var_name} in {var_mip} table")
+        glob_attrs['mip'] = var_info['mip']
+        cmor_info = cmor_table.get_variable(var_info['mip'], var_name)
+        _extract_variable(var, var_info, cmor_info, glob_attrs, filepath,
+                          out_dir)
+        logger.info(f"CMORization of {var_name} in {var_mip} table was "
+                    "successful")

esmvaltool/recipes/examples/recipe_check_obs.yml

@@ -509,6 +509,21 @@ diagnostics:
     scripts: null
 
 
+  HadEX3:
+    description: HadEX3 check
+    variables:
+      txx:
+      tnn:
+      rx1day:
+      rx5day:
+    additional_datasets:
+      - {dataset: HadEX3, project: OBS6, mip: yr, tier: 2,
+         type: ground, version: v3.0.4, start_year: 1901, end_year: 2018}
+      - {dataset: HadEX3, project: OBS6, mip: Amon, tier: 2,
+         type: ground, version: v3.0.4, start_year: 1901, end_year: 2018}
+    scripts: null
+
+
   HadISST:
     description: HadISST check
     variables:

esmvaltool/references/hadex3.bibtex

@@ -0,0 +1,12 @@
+@article{hadex3,
+  title = {Development of an updated global land in situ-based data set of temperature and precipitation extremes: HadEX3},
+  author = {Dunn, Robert JH and Alexander, Lisa V and Donat, Markus G and Zhang, Xuebin and Bador, Margot and Herold, Nicholas and Lippmann, Tanya and Allan, Rob and Aguilar, Enric and Barry, Abdoul Aziz and others},
+  doi = {10.1029/2019jd032263},
+  url = {https://doi.org/10.1029/2019JD032263},
+  journal = {Journal of Geophysical Research: Atmospheres},
+  volume = {125},
+  number = {16},
+  pages = {e2019JD032263},
+  year = {2020},
+  publisher = {Wiley Online Library}
+}