Merge branch 'main' into python312

ESMValGroup · Jun 5, 2024 · e10165f · e10165f
2 parents 5908854 + 75bc55c
commit e10165f
Show file tree

Hide file tree

Showing 12 changed files with 219 additions and 131 deletions.
diff --git a/.github/workflows/test-development.yml b/.github/workflows/test-development.yml
@@ -12,7 +12,6 @@ on:
   push:
     branches:
       - main
-      - fix_recipe_filler_bkwds_incompatibility
   schedule:
     - cron: '0 0 * * *'
 

diff --git a/conda-linux-64.lock b/conda-linux-64.lock
diff --git a/doc/sphinx/source/input.rst b/doc/sphinx/source/input.rst
@@ -271,7 +271,7 @@ A list of the datasets for which a CMORizers is available is provided in the fol
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | CowtanWay                    | tasa (Amon)                                                                                          |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
-| CRU                          | tas, pr (Amon)                                                                                       |   2  | Python          |
+| CRU                          | tas, tasmin, tasmax,  pr, clt (Amon), evspsblpot (Emon)                                              |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | CT2019                       | co2s (Amon)                                                                                          |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+

diff --git a/environment.yml b/environment.yml
@@ -21,14 +21,14 @@ dependencies:
   - distributed
   - ecmwf-api-client
   - eofs
-  - esmpy !=8.1.0,<8.6.0  # https://github.com/SciTools-incubator/iris-esmf-regrid/pull/342#issuecomment-2092921514
+  - esmpy >=8.6.0  # github.com/SciTools-incubator/iris-esmf-regrid/pull/342
   # - esmvalcore 2.10.*
   - fiona
   - fire
   - fsspec
   - gdal
   - iris >=3.6.1
-  - iris-esmf-regrid >=0.7.0
+  - iris-esmf-regrid >=0.10.0  # github.com/SciTools-incubator/iris-esmf-regrid/pull/342
   - jinja2
   - joblib
   - lime

diff --git a/environment_osx.yml b/environment_osx.yml
@@ -21,14 +21,14 @@ dependencies:
   - distributed
   - ecmwf-api-client
   - eofs
-  - esmpy !=8.1.0,<8.6.0  # https://github.com/SciTools-incubator/iris-esmf-regrid/pull/342#issuecomment-2092921514
+  - esmpy >=8.6.0  # github.com/SciTools-incubator/iris-esmf-regrid/pull/342
   # - esmvalcore 2.10.*
   - fiona
   - fsspec
   - fire
   - gdal
   - iris >=3.6.1
-  - iris-esmf-regrid >=0.7.0
+  - iris-esmf-regrid >=0.10.0  # github.com/SciTools-incubator/iris-esmf-regrid/pull/342
   - jinja2
   - joblib
   - lime
@@ -45,20 +45,12 @@ dependencies:
   - pip !=21.3
   - progressbar2
   - prov
-<<<<<<< HEAD
   - psyplot >=1.5.0  # earliest Py312 build
   - psy-maps >=1.5.0  # earliest Py312 build
   # - psy-reg >=1.5.0  # earliest Py312 build; not yet at 1.5.0 (4 Apr 2024); needs support for Python 3.12
   - psy-simple >=1.5.0  # earliest Py312 build
   - pyproj >=2.1
-=======
-  - psyplot
-  - psy-maps
-  - psy-reg
-  - psy-simple
-  - pyproj >=2.1
   - pys2index  # only from conda-forge
->>>>>>> main
   - python >=3.9
   - python-cdo
   - python-dateutil

diff --git a/esmvaltool/cmorizers/data/cmor_config/CRU.yml b/esmvaltool/cmorizers/data/cmor_config/CRU.yml
@@ -1,15 +1,20 @@
 ---
 # Filename (will be extended by variable name)
-filename: 'cru_ts4.02.1901.2017.{raw_name}.dat.nc'
+# filename: 'cru_ts4.02.1901.2017.{raw_name}.dat.nc'
+filename: 'cru_ts4.07.1901.2022.{raw_name}.dat.nc'
 
 # Common global attributes for Cmorizer output
 attributes:
   dataset_id: CRU
-  version: 'TS4.02'
+  # version: TS4.02
+  version: 'TS4.07'
   tier: 2
   modeling_realm: reanaly
-  project_id: OBS
-  source: 'https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.02/cruts.1811131722.v4.02/'
+  # project_id: OBS  # v4.02
+  project_id: OBS6
+  # source: 'https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.02/cruts.1811131722.v4.02/'
+  # source: 'https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.06/cruts.2205201912.v4.06/'
+  source: 'https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.07/cruts.2304141047.v4.07/'
   reference: 'cru'
   comment: ''
 
@@ -19,7 +24,23 @@ variables:
     mip: Amon
     raw: tmp
     raw_units: celsius
+  tasmin:
+    mip: Amon
+    raw: tmn
+    raw_units: celsius
+  tasmax:
+    mip: Amon
+    raw: tmx
+    raw_units: celsius
   pr:
     mip: Amon
     raw: pre
     raw_units: kg m-2 month-1
+  evspsblpot:
+    mip: Emon
+    raw: pet
+    raw_units: kg m-2 day-1
+  clt:
+    mip: Amon
+    raw: cld
+    raw_units: percent
diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml
@@ -281,12 +281,19 @@ datasets:
 
   CRU:
     tier: 2
-    source: https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.02/cruts.1811131722.v4.02/
-    last_access: 2019-05-16
+    source: https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.07/cruts.2304141047.v4.07/
+    last_access: 2023-11-06
     info: |
-      Download the following files:
-        {raw_name}/cru_ts4.02.1901.2017.{raw_name}.dat.nc.gz
+      Files can be downloaded using the download script (latest version only)
+      or manually:
+        {raw_name}/cru_ts4.07.1901.2022.{raw_name}.dat.nc.gz
       where {raw_name} is the name of the desired variable(s).
+      Previous versions can be downloaded from the corresponding folders in
+      https://crudata.uea.ac.uk/cru/data/hrg/. ESMValTools formatter can be used
+      for older versions with minor adjustments of
+      ``esmvaltool/cmorizers/data/cmor_config/CRU.yml``
+      Exact time coordinates and number of stations are not available version
+      TS4.02.
 
   CT2019:
     tier: 2

diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/cru.py b/esmvaltool/cmorizers/data/downloaders/datasets/cru.py
@@ -7,8 +7,9 @@
 logger = logging.getLogger(__name__)
 
 
-def download_dataset(config, dataset, dataset_info, start_date, end_date,
-                     overwrite):
+def download_dataset(
+    config, dataset, dataset_info, start_date, end_date, overwrite
+):
     """Download dataset.
 
     Parameters
@@ -32,10 +33,18 @@ def download_dataset(config, dataset, dataset_info, start_date, end_date,
         dataset_info=dataset_info,
         overwrite=overwrite,
     )
-    for var in ['tmp', 'pre']:
+    for var in ['tmp', 'pre', 'pet', 'tmn', 'tmx', 'cld']:
         downloader.download_file(
-            'https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.02/'
-            f'cruts.1811131722.v4.02/{var}/'
-            f'cru_ts4.02.1901.2017.{var}.dat.nc.gz',
-            wget_options=[])
+            "https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.07/"
+            f"cruts.2304141047.v4.07/{var}/"
+            f"cru_ts4.07.1901.2022.{var}.dat.nc.gz",
+            wget_options=[],
+        )
+    # for var in ['tmp', 'pre']:  # v TS4.02
+    #     downloader.download_file(
+    #         "https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.02/"
+    #         f"cruts.1811131722.v4.02/{var}/"
+    #         f"cru_ts4.02.1901.2017.{var}.dat.nc.gz",
+    #         wget_options=[],
+    #     )
     unpack_files_in_folder(downloader.local_folder)
diff --git a/esmvaltool/cmorizers/data/formatters/datasets/cru.py b/esmvaltool/cmorizers/data/formatters/datasets/cru.py
@@ -4,48 +4,83 @@
     Tier 2: other freely-available dataset.
 
 Source
-    https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.02/cruts.1811131722.v4.02/
+    TS4.02: https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.02/cruts.1811131722.v4.02/  # noqa: E501
+    TS4.06: https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.06/cruts.2205201912.v4.06/  # noqa: E501
+    TS4.07: https://crudata.uea.ac.uk/cru/data/hrg/cru_ts_4.07/cruts.2304141047.v4.07/  # noqa: E501
 
 Last access
-    20190516
+    TS4.02: 20190516
+    TS4.06: 20231012
+    TS4.07: 20231012
 
 Download and processing instructions
     Download the following files:
-        {raw_name}/cru_ts4.02.1901.2017.{raw_name}.dat.nc.gz
-    where {raw_name} is the name of the desired variable(s).
+    ``{raw_name}/cru_ts4.{X}.1901.{end_year}.{raw_name}.dat.nc.gz``
+    where ``{raw_name}`` is the name of the desired variable(s) or run
+    ``esmvaltool data download CRU`` for the latest version
 """
 
 import logging
 import os
 
+import cftime
 import iris
+import numpy as np
+from cf_units import Unit
 from iris import NameConstraint
 
 from esmvaltool.cmorizers.data import utilities as utils
 
 logger = logging.getLogger(__name__)
 
 
+def _center_timecoord(cube):
+    """Set time coordinates to exact center of each month.
+
+    CRU timepoints are not in the center of the month and added bounds
+    by utils.fix_coords are incorrect. #1981
+    """
+    time = cube.coord("time")
+    times = time.units.num2date(time.points)
+
+    # get bounds
+    starts = [cftime.DatetimeNoLeap(c.year, c.month, 1) for c in times]
+    ends = [
+        cftime.DatetimeNoLeap(c.year, c.month + 1, 1)
+        if c.month < 12 else cftime.DatetimeNoLeap(c.year + 1, 1, 1)
+        for c in times
+    ]
+    time.bounds = time.units.date2num(np.stack([starts, ends], -1))
+    time.points = [np.mean((t1, t2)) for t1, t2 in time.bounds]
+
+
 def _extract_variable(short_name, var, cfg, filepath, out_dir):
     """Extract variable."""
-    raw_var = var.get('raw', short_name)
+    raw_var = var.get("raw", short_name)
+    version = cfg["attributes"]["version"]
     cube = iris.load_cube(filepath, NameConstraint(var_name=raw_var))
 
     # Fix units
-    if 'raw_units' in var:
-        cube.units = var['raw_units']
-    cmor_info = cfg['cmor_table'].get_variable(var['mip'], short_name)
+    if "raw_units" in var:
+        cube.units = var["raw_units"]
+    cmor_info = cfg["cmor_table"].get_variable(var["mip"], short_name)
     cube.convert_units(cmor_info.units)
-    utils.convert_timeunits(cube, 1950)
+    if version in ["TS4.02"]:
+        utils.convert_timeunits(cube, 1950)
+    else:
+        cube.coord("time").convert_units(
+            Unit("days since 1950-1-1 00:00:00", calendar="gregorian"))
 
     # Fix coordinates
     utils.fix_coords(cube)
-    if 'height2m' in cmor_info.dimensions:
+    if "height2m" in cmor_info.dimensions:
         utils.add_height2m(cube)
+    if version not in ["TS4.02"]:
+        _center_timecoord(cube)
 
     # Fix metadata
-    attrs = cfg['attributes']
-    attrs['mip'] = var['mip']
+    attrs = cfg["attributes"]
+    attrs["mip"] = var["mip"]
     utils.fix_var_metadata(cube, cmor_info)
     utils.set_global_atts(cube, attrs)
 
@@ -54,17 +89,17 @@ def _extract_variable(short_name, var, cfg, filepath, out_dir):
                         short_name,
                         out_dir,
                         attrs,
-                        unlimited_dimensions=['time'])
+                        unlimited_dimensions=["time"])
 
 
 def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date):
     """Cmorization func call."""
-    raw_filepath = os.path.join(in_dir, cfg['filename'])
+    raw_filepath = os.path.join(in_dir, cfg["filename"])
 
     # Run the cmorization
-    for (short_name, var) in cfg['variables'].items():
+    for short_name, var in cfg["variables"].items():
         logger.info("CMORizing variable '%s'", short_name)
-        raw_var = var.get('raw', short_name)
+        raw_var = var.get("raw", short_name)
         filepath = raw_filepath.format(raw_name=raw_var)
         if filepath is None:
             continue

diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml
@@ -64,11 +64,33 @@ diagnostics:
   CRU:
     description: CRU check
     variables:
-      tas:
-      pr:
-    additional_datasets:
-      - {dataset: CRU, project: OBS, mip: Amon, tier: 2,
-         type: reanaly, version: TS4.02, start_year: 1901, end_year: 2017}
+      tas:  # check older versions
+        mip: Amon
+        additional_datasets:
+        - {dataset: CRU, project: OBS, tier: 2, type: reanaly, 
+          version: TS4.02, start_year: 1901, end_year: 2017}
+        - {dataset: CRU, project: OBS6, tier: 2, type: reanaly, 
+          version: TS4.07, start_year: 1901, end_year: 2021}
+      pr:  # check older versions
+        mip: Amon
+        additional_datasets:
+        - {dataset: CRU, project: OBS, tier: 2, type: reanaly, 
+        version: TS4.02, start_year: 1901, end_year: 2017}
+        - {dataset: CRU, project: OBS6, tier: 2, type: reanaly, 
+          version: TS4.07, start_year: 1901, end_year: 2021}
+      tasmin:
+        mip: Amon
+      tasmax:
+        mip: Amon
+      clt:
+        mip: Amon
+      evspsblpot:
+        mip: Emon
+    additional_datasets:  # newest version for all variables
+      - {dataset: CRU, project: OBS6, tier: 2,
+         type: reanaly, version: TS4.07, start_year: 1901, end_year: 2022}
+
+
     scripts: null
 
 

diff --git a/esmvaltool/references/cru.bibtex b/esmvaltool/references/cru.bibtex
@@ -1,13 +1,16 @@
 @article{cru,
-	doi = {10.1002/joc.3711},
-	url = {https://doi.org/10.1002%2Fjoc.3711},
-	year = 2013,
-	month = {may},
-	publisher = {Wiley},
-	volume = {34},
-	number = {3},
-	pages = {623--642},
-	author = {I. Harris and P.D. Jones and T.J. Osborn and D.H. Lister},
-	title = {Updated high-resolution grids of monthly climatic observations - the {CRU} {TS}3.10 Dataset},
-	journal = {International Journal of Climatology}
-}
+  title = {Version 4 of the {{CRU TS}} Monthly High-Resolution Gridded Multivariate Climate Dataset},
+  author = {Harris, Ian and Osborn, Timothy J. and Jones, Phil and Lister, David},
+  date = {2020-04-03},
+  year = 2020,
+  month = {april},
+  journaltitle = {Sci Data},
+  volume = {7},
+  number = {1},
+  pages = {109},
+  publisher = {{Nature Publishing Group}},
+  issn = {2052-4463},
+  doi = {10.1038/s41597-020-0453-3},
+  url = {https://www.nature.com/articles/s41597-020-0453-3},
+  urldate = {2023-10-12},
+}
diff --git a/setup.py b/setup.py
@@ -31,9 +31,9 @@
         'distributed',
         'ecmwf-api-client',
         'eofs',
-        'ESMPy',
+        'ESMPy',  # not on PyPI
         # 'esmvalcore',
-        'esmf-regrid>=0.7.0',
+        'esmf-regrid>=0.10.0',  # iris-esmf-regrid #342
         'fiona',
         'fire',
         'fsspec',