Cci lst v3 uncert diagnostic

esmvaltool/cmorizers/data/cmor_config/ESACCI-LST.yml

@@ -1,31 +1,102 @@
 # CMORIZE ESA CCI LST
-# Follwing CRU and ESACCI-OC as examples
-# Only looking at AQUA MONTHLY data but python has placeholders for different platforms
+# Only looking at AQUA MONTHLY
 ---
 
-
 # Common global attributes for Cmorizer output
 attributes:
   dataset_id: ESACCI-LST
-  version: '1.00'
+  version: '3.00'
   tier: 2
   project_id: OBS
   source: 'ESA CCI'
   modeling_realm: sat
   reference: 'esacci_lst'
   comment: ''
-  start_year: 2003
-  end_year: 2018
+  start_year: 2006
+  end_year: 2007
 
 # Variables to cmorize
 # These go into the vals dictionary in the python script
 variables:
-  ts:
+  tsDay:
+    mip: Amon
+    raw: land surface temperature
+    raw_units: kelvin
+    file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-'
+    start_year: 2006
+    end_year: 2007
+    short_name: XYZ_TESTING  
+
+
+  tsNight:
     mip: Amon
     raw: land surface temperature
     raw_units: kelvin
-    file_day: 'ESACCI-LST-L3C-LST-MODISA-0.05deg_1MONTHLY_DAY-'
-    file_night: 'ESACCI-LST-L3C-LST-MODISA-0.05deg_1MONTHLY_NIGHT-'
-    # plan to make it possible to change MODISA here but only placeholders in the python there for now
-    # Period covered: 2003-2018
-    # 2003 Only has data from July onwards for Aqua MODIS
+    file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-'
+    start_year: 2006
+    end_year: 2007
+
+
+  tsLocalAtmErrDay:    
+    mip: Amon
+    raw: uncertainty from locally correlated errors on atmospheric scales
+    raw_units: kelvin
+    file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-'
+    start_year: 2006
+    end_year: 2007
+
+  tsLocalAtmErrNight:    
+    mip: Amon
+    raw: uncertainty from locally correlated errors on atmospheric scales
+    raw_units: kelvin
+    file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-'
+    start_year: 2006
+    end_year: 2007
+
+  tsLocalSfcErrDay:    
+    mip: Amon
+    raw: uncertainty from locally correlated errors on surface scales
+    raw_units: kelvin
+    file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-'
+    start_year: 2006
+    end_year: 2007
+
+  tsLocalSfcErrNight:    
+    mip: Amon
+    raw: uncertainty from locally correlated errors on surface scales
+    raw_units: kelvin
+    file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-'
+    start_year: 2006
+    end_year: 2007
+
+  tsLSSysErrDay:    
+    mip: Amon
+    raw: uncertainty from large-scale systematic errors
+    raw_units: kelvin
+    file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-'
+    start_year: 2006
+    end_year: 2007
+
+  tsLSSysErrNight:    
+    mip: Amon
+    raw: uncertainty from large-scale systematic errors
+    raw_units: kelvin
+    file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-'
+    start_year: 2006
+    end_year: 2007
+
+  tsUnCorErrDay:    
+    mip: Amon
+    raw: uncertainty from uncorrelated errors
+    raw_units: kelvin
+    file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_DAY-'
+    start_year: 2006
+    end_year: 2007
+
+  tsUnCorErrNight:    
+    mip: Amon
+    raw: uncertainty from uncorrelated errors
+    raw_units: kelvin
+    file: 'ESACCI-LST-L3C-LST-MODISA-0.01deg_1MONTHLY_NIGHT-'
+    start_year: 2006
+    end_year: 2007

esmvaltool/cmorizers/data/formatters/datasets/esacci_lst.py

@@ -1,160 +1,117 @@
-"""ESMValTool CMORizer for ESACCI-LST data.
-
-Tier
-   Tier 2: other freely-available dataset.
-
-Source
-   On CEDA-JASMIN
-   /gws/nopw/j04/esacci_lst/public
-   For access to this JASMIN group workspace please register at
-   https://accounts.jasmin.ac.uk/services/group_workspaces/esacci_lst/
-
-Download and processing instructions
-   Put all files under a single directory (no subdirectories with years)
-   in ${RAWOBS}/Tier2/ESACCI-LST
-   BOTH DAY and NIGHT files are needed for each month
+"""ESMValTool CMORizer for ESACCI-LST-UNCERT data.
 
+BOTH DAY and NIGHT files are needed for each month
 Currently set to work with only the MODIS AQUA L3 monthly data
-
-Modification history
-   20201015 Started by Robert King
-   20201029 Day/Night averaging added along with CMOR utils
 """
 
 import datetime
 import logging
-from calendar import monthrange
-
 import iris
+import cf_units as unit
 
-from esmvaltool.cmorizers.data import utilities as utils
+from ...utilities import fix_coords
 
 logger = logging.getLogger(__name__)
 
 
 def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date):
     """Cmorization func call."""
-    cmor_table = cfg['cmor_table']
     glob_attrs = cfg['attributes']
 
-    # run the cmorization
+    # variable_list contains the variable list
+    # variable_keys has the short 'code' as a key for the variables.
+    # both these lists are in made in the same order
 
     # vals has the info from the yml file
     # var is set up in the yml file
+    print('........................')
+    print(cfg['variables'])
     for var, vals in cfg['variables'].items():
-        # leave this loop in as might be useful in
-        # the future for getting other info
-        # like uncertainty information from the original files
-
         glob_attrs['mip'] = vals['mip']
-        cmor_info = cmor_table.get_variable(vals['mip'], var)
-        var_name = cmor_info.short_name
-
-        for key in vals.keys():
-            logger.info("%s %s", key, vals[key])
-
-        variable = vals['raw']
-        # not currently used, but referenced for future
-        # platform = 'MODISA'
 
         # loop over years and months
         # get years from start_year and end_year
-        # note 2003 doesn't start until July so not included at this stage
-        for year in range(glob_attrs['start_year'],
-                          glob_attrs['end_year'] + 1):
-            this_years_cubes = iris.cube.CubeList()
-            for month0 in range(12):  # Change this in final version
-                month = month0 + 1
+        for year in range(vals['start_year'], vals['end_year'] + 1):
+            for month in range(1, 13):
+                logger.info(year)
                 logger.info(month)
-                day_cube, night_cube = load_cubes(in_dir, vals['file_day'],
-                                                  vals['file_night'], year,
-                                                  month, variable)
-
-                monthly_cube = make_monthly_average(day_cube, night_cube, year,
-                                                    month)
-
-                # use CMORizer utils
-                monthly_cube = utils.fix_coords(monthly_cube)
-
-                this_years_cubes.append(monthly_cube)
-
-            # Use utils save
-            # This seems to save files all with the same name!
-            # Fixed by making yearly files
-            this_years_cubes = this_years_cubes.merge_cube()
-            this_years_cubes.long_name = 'Surface Temperature'
-            this_years_cubes.standard_name = 'surface_temperature'
-
-            # Fix variable metadata
-            utils.fix_var_metadata(this_years_cubes, cmor_info)
-
-            # Fix global metadata
-            utils.set_global_atts(this_years_cubes, glob_attrs)
-
-            utils.save_variable(
-                this_years_cubes,
-                var_name,
-                out_dir,
-                glob_attrs,
-                unlimited_dimensions=['time'],
-            )
-
-
-def load_cubes(in_dir, file_day, file_night, year, month, variable):
-    """Variable description.
-
-    variable = land surface temperature
-    platform = AQUA not used for now
-               but in place for future expansion to all ESC CCI LST platforms
-    """
-    logger.info('Loading %s/%s%s%s*.nc', in_dir, file_day, year, month)
-    day_cube = iris.load_cube(
-        '%s/%s%s%02d*.nc' % (in_dir, file_day, year, month), variable)
-    logger.info('Loading %s/%s%s%s*.nc', in_dir, file_night, year, month)
-    night_cube = iris.load_cube(
-        '%s/%s%s%02d*.nc' % (in_dir, file_night, year, month), variable)
-
-    return day_cube, night_cube
-
-
-def make_monthly_average(day_cube, night_cube, year, month):
-    """Make the average LST form the day time and night time files."""
-    day_cube.attributes.clear()
-    night_cube.attributes.clear()
-
-    co_time = night_cube.coord('time')
-    co_time.points = co_time.points + 100.0
-    # maybe the arbitrary difference should go on day cubes to
-    # take the timestamp to 12Z?
-    # not really an issue when using monthly files
-
-    result = iris.cube.CubeList([day_cube, night_cube]).concatenate_cube()
-
-    # This corrects the lonitude coord name issue
-    # This should be fixed in the next version of the CCI data
-    logger.info("Longitude coordinate correction being applied")
-    result.coords()[2].var_name = 'longitude'
-    result.coords()[2].standard_name = 'longitude'
-    result.coords()[2].long_name = 'longitude'
-
-    monthly_cube = result.collapsed('time', iris.analysis.MEAN)
-
-    # fix time coordinate bounds
-    monthly_co_time = monthly_cube.coord('time')
-
-    time_point = (datetime.datetime(year, month, 1, 0, 0) -
-                  datetime.datetime(1981, 1, 1, 0, 0, 0)).total_seconds()
-    monthly_co_time.points = time_point
-
-    num_days = monthrange(year, month)[1]
-    monthly_co_time.bounds = [
-        time_point, time_point + ((num_days - 1) * 24 * 3600)
-    ]
-    # should this be num_days or num_days-1 ### question for Valeriu or Axel
-    # or 23:59:59 ???
-
-    monthly_cube.attributes = {
-        'information': 'Mean of Day and Night Aqua MODIS monthly LST'
-    }
-
-    return monthly_cube
+                cubes = load_cubes(in_dir,
+                                   vals['file'],
+                                   year,
+                                   month,
+                                   vals['raw'],
+                                   )
+
+                #  make time coords
+                time_units = 'hours since 1970-01-01 00:00:00'
+                time_point = unit.date2num(datetime.datetime(year, month, 1),
+                                           time_units,
+                                           unit.CALENDAR_STANDARD
+                                           )
+
+                time_coord = iris.coords.DimCoord(time_point,
+                                                  standard_name='time',
+                                                  long_name='time',
+                                                  var_name='time',
+                                                  units=time_units,
+                                                  bounds=None,
+                                                  attributes=None,
+                                                  coord_system=None,
+                                                  circular=False
+                                                  )
+
+                cubes.attributes = {}
+                cubes.attributes['var'] = var
+
+                try:
+                    cubes.remove_coord('time')
+                except:
+                    logger.info('Coord fix issue %s' % cubes.long_name)
+
+                cubes.add_dim_coord(time_coord, 0)
+
+                if cubes.long_name == 'land surface temperature':
+                    cubes.long_name = 'surface_temperature'
+                    cubes.standard_name = 'surface_temperature'
+
+                try:
+                    cubes = fix_coords(cubes)
+                except:
+                    logger.info('skip fixing')
+                    logger.info(cubes.long_name)
+
+                try:
+                    cubes.coords()[2].standard_name = 'longitude'
+                except:
+                    # No change needed
+                    pass
+
+                var_name = cubes.attributes['var']
+
+                if cubes.var_name == 'lst':
+                    cubes.var_name = 'ts'
+
+                if 'Day' in var_name:
+                    cubes.long_name += ' Day'
+                    cubes.var_name += '_day'
+
+
+                if 'Night' in var_name:
+                    cubes.long_name += ' Night'
+                    cubes.var_name += '_night'
+
+                # make use - and _ consistent!!!
+                save_name = f'{out_dir}/OBS_ESACCI-LST_sat_3.00_Amon_{var_name}_{year}{month:02d}.nc'
+                iris.save(cubes,
+                          save_name
+                          )
+
+
+def load_cubes(in_dir, file, year, month, variable_list):
+    """Load files into cubes based on variables wanted in variable_list."""
+    logger.info(f'Loading {in_dir}/{file}{year}{month:02d}.nc')
+    cube = iris.load_cube(f'{in_dir}/{file}{year}{month:02d}*.nc',
+                          variable_list
+                          )
+
+    return cube