Add CMORizer script for NSIDC-G02202-sh sea ice fraction (#3512)

Co-authored-by: Axel Lauer <[email protected]>
Co-authored-by: Valeriu Predoi <[email protected]>

doc/sphinx/source/input.rst

@@ -406,6 +406,8 @@ A list of the datasets for which a CMORizers is available is provided in the fol
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | NSIDC-0116-[nh|sh] [#note4]_ | usi, vsi (day)                                                                                       |   3  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
+| NSIDC-g02202-[sh]            | siconc (SImon)                                                                                       |   3  | Python          |
++------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | OceanSODA-ETHZ               | areacello (Ofx), co3os, dissicos, fgco2, phos, spco2, talkos (Omon)                                  |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | OSI-450-[nh|sh]              | sic (OImon), sic (day)                                                                               |   2  | Python          |

esmvaltool/cmorizers/data/cmor_config/NSIDC-G02202-sh.yml

@@ -0,0 +1,23 @@
+---
+filename: seaice_conc_monthly_sh_{year}.*.nc
+# Common global attributes for Cmorizer output
+attributes:
+  dataset_id: NSIDC-G02202-sh
+  version: '4'
+  tier: 3
+  modeling_realm: reanaly
+  project_id: OBS6
+  source: 'https://nsidc.org/data/g02202/versions/4'
+  reference: 'nsidc-g02202'
+  comment: ''
+
+variables:
+  siconc:
+    mip: SImon
+    raw: cdr_seaice_conc_monthly 
+    compress: true
+
+
+custom:
+  create_areacello: true
+  area_file: pss25area_v3.dat

esmvaltool/cmorizers/data/datasets.yml

@@ -1069,6 +1069,14 @@ datasets:
       Download daily data from:
       https://nsidc.org/data/NSIDC-0116
       Login required for download, and also requires citation only to use
+   
+  NSIDC-G02202-sh:
+    tier: 3
+    source: https://polarwatch.noaa.gov/erddap/griddap/nsidcG02202v4shmday
+    last_access: 2023-05-13
+    info: |
+      Download monthly data. 
+      Login required for download, and also requires citation only to use
 
   OceanSODA-ETHZ:
     tier: 2

esmvaltool/cmorizers/data/downloaders/datasets/nsidc_g02202_sh.py

@@ -0,0 +1,80 @@
+"""Script to download NSIDC-G02202-sh."""
+import logging
+from datetime import datetime
+from dateutil import relativedelta
+
+from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader
+
+logger = logging.getLogger(__name__)
+
+
+def download_dataset(config, dataset, dataset_info, start_date, end_date,
+                     overwrite):
+    """Download dataset.
+
+    Parameters
+    ----------
+    config : dict
+        ESMValTool's user configuration
+    dataset : str
+        Name of the dataset
+    dataset_info : dict
+         Dataset information from the datasets.yml file
+    start_date : datetime
+        Start of the interval to download
+    end_date : datetime
+        End of the interval to download
+    overwrite : bool
+        Overwrite already downloaded files
+    """
+    if start_date is None:
+        start_date = datetime(1979, 1, 1)
+    if end_date is None:
+        end_date = datetime(2023, 1, 1)
+
+    loop_date = start_date
+
+    downloader = WGetDownloader(
+        config=config,
+        dataset=dataset,
+        dataset_info=dataset_info,
+        overwrite=overwrite,
+    )
+
+    # need area file
+    area_dat = ('ftp://sidads.colorado.edu/DATASETS/seaice'
+                '/polar-stereo/tools/pss25area_v3.dat')
+    downloader.download_folder(area_dat, [])
+
+    anc_path = ('https://noaadata.apps.nsidc.org/NOAA/G02202_V4/'
+                'ancillary/G02202-cdr-ancillary-sh.nc')
+    downloader.download_folder(anc_path, [])
+
+    base_path = ('https://noaadata.apps.nsidc.org/NOAA/G02202_V4/south/monthly'
+                 '/seaice_conc_monthly_sh_{year}{month:02d}_{other}_v04r00.nc')
+
+    # regex for n07 changes to f08.. file names
+    # bins #{'197811':'n07','198708':'f08',
+    # '199201':'f11','199510':'f13', '200801':'f17'}
+    datels = [datetime(1978, 11, 1), datetime(1987, 7, 30),
+              datetime(1991, 12, 30), datetime(1995, 9, 30),
+              datetime(2007, 12, 30), end_date]
+    suffls = ['n07', 'f08', 'f11', 'f13', 'f17']
+    isuf = 0
+    suffix = suffls[isuf]
+    # initialize suffix if dates start higher than initial
+    while loop_date >= datels[isuf]:
+        suffix = suffls[isuf]
+        isuf += 1
+
+    while loop_date <= end_date:
+
+        if loop_date > datels[isuf]:
+            suffix = suffls[isuf]
+            isuf += 1
+
+        downloader.download_folder(
+            base_path.format(year=loop_date.year, month=loop_date.month,
+                             other=suffix), [])
+        loop_date += relativedelta.relativedelta(months=1)
+        # check loop_date is => next bin

esmvaltool/cmorizers/data/formatters/datasets/nsidc_g02202_sh.py

@@ -0,0 +1,182 @@
+"""ESMValTool CMORizer for Sea Ice Concentration CDR.
+
+Tier
+   Tier 3: restricted dataset.
+
+Source
+   https://nsidc.org/data/g02202/versions/4
+
+Last access
+   20231213
+
+Download and processing instructions
+    Download data from:
+    https://noaadata.apps.nsidc.org/NOAA/G02202_V4/south/monthly
+    lat and lon from:
+    https://noaadata.apps.nsidc.org/NOAA/G02202_V4/ancillary/
+    area file:
+    ftp://sidads.colorado.edu/DATASETS/seaice/polar-stereo/tools/
+    pss25area_v3.dat
+
+    https://nsidc.org/sites/default/files/g02202-v004-userguide_1_1.pdf
+
+"""
+
+import logging
+import os
+import re
+
+import numpy as np
+
+import iris
+from cf_units import Unit
+from iris.coords import AuxCoord
+
+from esmvaltool.cmorizers.data import utilities as utils
+
+logger = logging.getLogger(__name__)
+
+
+def _get_filepaths(in_dir, basename, yyyy):
+    """Find correct name of file (extend basename with timestamp)."""
+    f_name = basename.format(year=yyyy)
+    regex = re.compile(f_name)
+    return_files = []
+    for files in os.listdir(in_dir):
+
+        if regex.match(files):
+            return_files.append(os.path.join(in_dir, files))
+
+    return return_files
+
+
+def _fix_time_coord(cube, _field, _filename):
+    """Set time points to central day of month."""
+    time_coord = cube.coord('time')
+    new_unit = Unit('days since 1850-01-01 00:00:00', calendar='standard')
+    time_coord.convert_units(new_unit)
+    old_time = new_unit.num2date(time_coord.points)
+    new_time = [d.replace(day=15) for d in old_time]
+    time_coord.points = new_unit.date2num(new_time)
+
+
+def _prom_dim_coord(cube, _field, _filename):
+    iris.util.promote_aux_coord_to_dim_coord(cube, 'time')
+
+
+def _create_coord(cubes, var_name, standard_name):
+    cube = cubes.extract_cube(standard_name)
+    coord = AuxCoord(
+        cube.data,
+        standard_name=standard_name,
+        long_name=cube.long_name,
+        var_name=var_name,
+        units='degrees'  # cube.units,
+    )
+    return coord
+
+
+def _extract_variable(raw_var, cmor_info, attrs, filepath, out_dir, latlon):
+    """Extract variable from all files."""
+    var = cmor_info.short_name
+    cubes = iris.load(filepath, raw_var, _prom_dim_coord)
+    iris.util.equalise_attributes(cubes)
+
+    cube = cubes.concatenate_cube()
+    iris.util.promote_aux_coord_to_dim_coord(cube, 'projection_y_coordinate')
+    iris.util.promote_aux_coord_to_dim_coord(cube, 'projection_x_coordinate')
+    cube.coord('projection_y_coordinate').rename('y')
+    cube.coord('projection_x_coordinate').rename('x')
+
+    cube.add_aux_coord(latlon[0], (1, 2))
+    cube.add_aux_coord(latlon[1], (1, 2))
+    # add coord typesi
+    area_type = AuxCoord([1.0], standard_name='area_type', var_name='type',
+                         long_name='Sea Ice area type')
+    cube.add_aux_coord(area_type)
+
+    # cube.convert_units(cmor_info.units)
+    cube.units = '%'
+    cube.data[cube.data > 100] = np.nan
+    cube = cube * 100
+
+    # utils.fix_coords(cube) #latlon multidimensional
+    utils.fix_var_metadata(cube, cmor_info)
+    utils.set_global_atts(cube, attrs)
+
+    utils.save_variable(cube,
+                        var,
+                        out_dir,
+                        attrs,
+                        unlimited_dimensions=['time'])
+
+    return cube
+
+
+def _create_areacello(cfg, in_dir, sample_cube, glob_attrs, out_dir):
+    if not cfg['custom'].get('create_areacello', False):
+        return
+    var_info = cfg['cmor_table'].get_variable('Ofx', 'areacello')
+    glob_attrs['mip'] = 'Ofx'
+    lat_coord = sample_cube.coord('latitude')
+
+    area_file = os.path.join(in_dir, cfg['custom']['area_file'])
+    with open(area_file, 'rb') as datfile:
+        areasdmnd = np.fromfile(datfile,
+                                dtype=np.int32).reshape(lat_coord.shape)
+
+    # Divide by 1000 to get km2 then multiply by 1e6 to m2 ...*1000
+    ardata = areasdmnd * 1000
+
+    cube = iris.cube.Cube(ardata,
+                          standard_name=var_info.standard_name,
+                          long_name=var_info.long_name,
+                          var_name=var_info.short_name,
+                          units='m2',
+                          dim_coords_and_dims=[(sample_cube.coord('y'), 0),
+                                               (sample_cube.coord('x'), 1)])
+    cube.add_aux_coord(lat_coord, (0, 1))
+    cube.add_aux_coord(sample_cube.coord('longitude'), (0, 1))
+    utils.fix_var_metadata(cube, var_info)
+    utils.set_global_atts(cube, glob_attrs)
+    utils.save_variable(cube, var_info.short_name, out_dir, glob_attrs,
+                        zlib=True)
+
+
+def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date):
+    """Cmorization func call."""
+    glob_attrs = cfg['attributes']
+    cmor_table = cfg['cmor_table']
+
+    # get aux nc file
+    cubesaux = iris.load(os.path.join(in_dir, 'G02202-cdr-ancillary-sh.nc'))
+    lat_coord = _create_coord(cubesaux, 'lat', 'latitude')
+    lon_coord = _create_coord(cubesaux, 'lon', 'longitude')
+    year = 1978
+    # split by year..
+    sample_cube = None
+    while year <= 2022:
+
+        filepaths = _get_filepaths(in_dir, cfg['filename'], year)
+
+        if len(filepaths) > 0:
+            logger.info("Found %d files in '%s'", len(filepaths), in_dir)
+
+            for (var, var_info) in cfg['variables'].items():
+                logger.info("CMORizing variable '%s'", var)
+                glob_attrs['mip'] = var_info['mip']
+                cmor_info = cmor_table.get_variable(var_info['mip'], var)
+                raw_var = var_info.get('raw', var)
+                sample_cube = _extract_variable(raw_var, cmor_info,
+                                                glob_attrs, filepaths,
+                                                out_dir, [lat_coord,
+                                                          lon_coord])
+
+        else:
+            logger.info("No files found ")
+            logger.info("year: %d basename: %s", year, cfg['filename'])
+
+        year += 1
+
+        if sample_cube is not None:
+            _create_areacello(cfg, in_dir, sample_cube, glob_attrs, out_dir)

esmvaltool/recipes/examples/recipe_check_obs.yml

@@ -1911,6 +1911,17 @@ diagnostics:
          type: reanaly, version: "4.1", start_year: 1978, end_year: 2018}
     scripts: null
 
+  NSIDC-G02202:
+    description: NSIDC-G02202 check
+    variables:
+      areacello:
+        mip: Ofx
+      siconc:
+        mip: SImon
+    additional_datasets:
+      - {dataset: NSIDC-G02202-sh, project: OBS6, tier: 3,
+        type: reanaly, version: 4, start_year: 1979, end_year: 2022}
+    scripts: null
 
   UWisc:
     description: UWisc check

esmvaltool/references/nsidc-g02202.bibtex

@@ -0,0 +1,8 @@
+@misc{nsidc-g02202,
+  doi = {10.7265/efmz-2t65},
+  title = {NOAA/NSIDC Climate Data Record of Passive Microwave Sea Ice Concentration, Version 4}, 
+  url = {https://nsidc.org/data/G02202/versions/4}, 
+  author = {Meier, W. N., F. Fetterer, A. K. Windnagel, and J. S. Stewart.},
+  publisher = {National Snow and Ice Data Center}, 
+  year = {2021} 
+}