Skip to content

Commit

Permalink
Also read default custom CMOR tables if custom location is specified (#…
Browse files Browse the repository at this point in the history 
…2279)

Co-authored-by: Valeriu Predoi <[email protected]>
  • Loading branch information
@schlunma @valeriupredoi
schlunma and valeriupredoi authored Jan 18, 2024
1 parent 9debbbc commit e03d320
Show file tree
Hide file tree
Showing 4 changed files with 199 additions and 89 deletions.
20 changes: 13 additions & 7 deletions doc/quickstart/configure.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -699,11 +699,18 @@ Custom CMOR tables
As mentioned in the previous section, the CMOR tables of projects that use
``cmor_strict: false`` will be extended with custom CMOR tables.
By default, these are loaded from `esmvalcore/cmor/tables/custom
For derived variables (the ones with ``derive: true`` in the recipe), the
custom CMOR tables will always be considered.
By default, these custom tables are loaded from `esmvalcore/cmor/tables/custom
<https://github.com/ESMValGroup/ESMValCore/tree/main/esmvalcore/cmor/tables/custom>`_.
However, by using the special project ``custom`` in the
``config-developer.yml`` file with the option ``cmor_path``, a custom location
for these custom CMOR tables can be specified:
for these custom CMOR tables can be specified.
In this case, the default custom tables are extended with those entries from
the custom location (in case of duplication, the custom location tables take
precedence).
Example:
.. code-block:: yaml
Expand Down Expand Up @@ -743,11 +750,10 @@ Example for the file ``CMOR_asr.dat``:
!----------------------------------
!
It is also possible to use a special coordinates file ``CMOR_coordinates.dat``.
If this is not present in the custom directory, the one from the default
directory (`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat
<https://github.com/ESMValGroup/ESMValCore/tree/main/esmvalcore/cmor/tables/custom/CMOR_coordinates.dat>`_)
is used.
It is also possible to use a special coordinates file ``CMOR_coordinates.dat``,
which will extend the entries from the default one
(`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat
<https://github.com/ESMValGroup/ESMValCore/tree/main/esmvalcore/cmor/tables/custom/CMOR_coordinates.dat>`_).
.. _filterwarnings_config-developer:
Expand Down
112 changes: 63 additions & 49 deletions esmvalcore/cmor/table.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ def get_variable(
self,
table_name: str,
short_name: str,
derived: Optional[bool] = False,
derived: bool = False,
) -> VariableInfo | None:
"""Search and return the variable information.
Expand Down Expand Up @@ -299,8 +299,8 @@ def get_variable(
# cmor_strict=False or derived=True
var_info = self._look_in_all_tables(derived, alt_names_list)

# If that didn' work either, look in default table if cmor_strict=False
# or derived=True
# If that didn't work either, look in default table if
# cmor_strict=False or derived=True
if not var_info:
var_info = self._look_in_default(derived, alt_names_list,
table_name)
Expand Down Expand Up @@ -988,51 +988,52 @@ class CustomInfo(CMIP5Info):
Parameters
----------
cmor_tables_path: str or None
cmor_tables_path:
Full path to the table or name for the table if it is present in
ESMValTool repository
"""

def __init__(self, cmor_tables_path=None):
cwd = os.path.dirname(os.path.realpath(__file__))
default_cmor_folder = os.path.join(cwd, 'tables', 'custom')
ESMValTool repository. If ``None``, use default tables from
`esmvalcore/cmor/tables/custom`.
# Get custom location of CMOR tables if possible
if cmor_tables_path is None:
self._cmor_folder = default_cmor_folder
else:
self._cmor_folder = self._get_cmor_path(cmor_tables_path)
if not os.path.isdir(self._cmor_folder):
raise ValueError(f"Custom CMOR tables path {self._cmor_folder} is "
f"not a directory")
"""

def __init__(self, cmor_tables_path: Optional[str | Path] = None) -> None:
"""Initialize class member."""
self.coords = {}
self.tables = {}
self.var_to_freq = {}
self.var_to_freq: dict[str, dict] = {}
table = TableInfo()
table.name = 'custom'
self.tables[table.name] = table

# Try to read coordinates from custom location, use default location if
# not possible
coordinates_file = os.path.join(
self._cmor_folder,
'CMOR_coordinates.dat',
)
if os.path.isfile(coordinates_file):
self._coordinates_file = coordinates_file
# First, read default custom tables from repository
self._cmor_folder = self._get_cmor_path('custom')
self._read_table_dir(self._cmor_folder)

# Second, if given, update default tables with user-defined custom
# tables
if cmor_tables_path is not None:
self._user_table_folder = self._get_cmor_path(cmor_tables_path)
if not os.path.isdir(self._user_table_folder):
raise ValueError(
f"Custom CMOR tables path {self._user_table_folder} is "
f"not a directory"
)
self._read_table_dir(self._user_table_folder)
else:
self._coordinates_file = os.path.join(
default_cmor_folder,
'CMOR_coordinates.dat',
)
self._user_table_folder = None

self.coords = {}
self._read_table_file(self._coordinates_file, self.tables['custom'])
for dat_file in glob.glob(os.path.join(self._cmor_folder, '*.dat')):
if dat_file == self._coordinates_file:
def _read_table_dir(self, table_dir: str) -> None:
"""Read CMOR tables from directory."""
# If present, read coordinates
coordinates_file = os.path.join(table_dir, 'CMOR_coordinates.dat')
if os.path.isfile(coordinates_file):
self._read_table_file(coordinates_file)

# Read other variables
for dat_file in glob.glob(os.path.join(table_dir, '*.dat')):
if dat_file == coordinates_file:
continue
try:
self._read_table_file(dat_file, self.tables['custom'])
self._read_table_file(dat_file)
except Exception:
msg = f"Exception raised when loading {dat_file}"
# Logger may not be ready at this stage
Expand All @@ -1042,29 +1043,40 @@ def __init__(self, cmor_tables_path=None):
print(msg)
raise

def get_variable(self, table, short_name, derived=False):
def get_variable(
self,
table: str,
short_name: str,
derived: bool = False
) -> VariableInfo | None:
"""Search and return the variable info.
Parameters
----------
table: str
Table name
short_name: str
Variable's short name
derived: bool, optional
table:
Table name. Ignored for custom tables.
short_name:
Variable's short name.
derived:
Variable is derived. Info retrieval for derived variables always
look on the default tables if variable is not find in the
requested table
looks on the default tables if variable is not found in the
requested table. Ignored for custom tables.
Returns
-------
VariableInfo
Return the VariableInfo object for the requested variable if
found, returns None if not
VariableInfo | None
`VariableInfo` object for the requested variable if found, returns
None if not.
"""
return self.tables['custom'].get(short_name, None)

def _read_table_file(self, table_file, table=None):
def _read_table_file(
self,
table_file: str,
_: Optional[TableInfo] = None,
) -> None:
"""Read a single table file."""
with open(table_file, 'r', encoding='utf-8') as self._current_table:
self._read_line()
while True:
Expand All @@ -1079,7 +1091,9 @@ def _read_table_file(self, table_file, table=None):
self.coords[value] = self._read_coordinate(value)
continue
elif key == 'variable_entry':
table[value] = self._read_variable(value, '')
self.tables['custom'][value] = self._read_variable(
value, ''
)
continue
if not self._read_line():
return
Expand Down
131 changes: 115 additions & 16 deletions tests/integration/cmor/test_read_cmor_tables.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from pathlib import Path
from textwrap import dedent

import pytest
import yaml
Expand All @@ -7,17 +8,6 @@
from esmvalcore.cmor.table import __file__ as root
from esmvalcore.cmor.table import read_cmor_tables

CUSTOM_CFG_DEVELOPER = {
'custom': {'cmor_path': Path(root).parent / 'tables' / 'custom'},
'CMIP6': {
'cmor_strict': True,
'input_dir': {'default': '/'},
'input_file': '*.nc',
'output_file': 'out.nc',
'cmor_type': 'CMIP6',
},
}


def test_read_cmor_tables_raiser():
"""Test func raiser."""
Expand Down Expand Up @@ -52,12 +42,108 @@ def test_read_cmor_tables():
assert Path(table._cmor_folder) == table_path / 'obs4mips' / 'Tables'
assert table.strict is False

project = 'custom'
table = CMOR_TABLES[project]
assert Path(table._cmor_folder) == table_path / 'custom'
assert table._user_table_folder is None
assert table.coords
assert table.tables['custom']


CMOR_NEWVAR_ENTRY = dedent(
"""
!============
variable_entry: newvarfortesting
!============
modeling_realm: atmos
!----------------------------------
! Variable attributes:
!----------------------------------
standard_name:
units: kg s m A
cell_methods: time: mean
cell_measures: area: areacella
long_name: Custom Variable for Testing
!----------------------------------
! Additional variable information:
!----------------------------------
dimensions: longitude latitude time
type: real
positive: up
!----------------------------------
!
"""
)
CMOR_NETCRE_ENTRY = dedent(
"""
!============
variable_entry: netcre
!============
modeling_realm: atmos
!----------------------------------
! Variable attributes:
!----------------------------------
standard_name: air_temperature ! for testing
units: K ! for testing
cell_methods: time: mean
cell_measures: area: areacella
long_name: This is New ! for testing
!----------------------------------
! Additional variable information:
!----------------------------------
dimensions: longitude latitude time
type: real
positive: up
!----------------------------------
!
"""
)
CMOR_NEWCOORD_ENTRY = dedent(
"""
!============
axis_entry: newcoordfortesting
!============
!----------------------------------
! Axis attributes:
!----------------------------------
standard_name:
units: kg
axis: Y ! X, Y, Z, T (default: undeclared)
long_name: Custom Coordinate for Testing
!----------------------------------
! Additional axis information:
!----------------------------------
out_name: newcoordfortesting
valid_min: -90.0
valid_max: 90.0
stored_direction: increasing
type: double
must_have_bounds: yes
!----------------------------------
!
"""
)


def test_read_custom_cmor_tables(tmp_path):
"""Test reading of custom CMOR tables."""
(tmp_path / 'CMOR_newvarfortesting.dat').write_text(CMOR_NEWVAR_ENTRY)
(tmp_path / 'CMOR_netcre.dat').write_text(CMOR_NETCRE_ENTRY)
(tmp_path / 'CMOR_coordinates.dat').write_text(CMOR_NEWCOORD_ENTRY)

custom_cfg_developer = {
'custom': {'cmor_path': str(tmp_path)},
'CMIP6': {
'cmor_strict': True,
'input_dir': {'default': '/'},
'input_file': '*.nc',
'output_file': 'out.nc',
'cmor_type': 'CMIP6',
},
}
cfg_file = tmp_path / 'config-developer.yml'
with cfg_file.open('w', encoding='utf-8') as file:
yaml.safe_dump(CUSTOM_CFG_DEVELOPER, file)
yaml.safe_dump(custom_cfg_developer, file)

read_cmor_tables(cfg_file)

Expand All @@ -66,10 +152,23 @@ def test_read_custom_cmor_tables(tmp_path):
assert 'custom' in CMOR_TABLES

custom_table = CMOR_TABLES['custom']
assert (Path(custom_table._cmor_folder) ==
Path(root).parent / 'tables' / 'custom')
assert (Path(custom_table._coordinates_file) ==
Path(root).parent / 'tables' / 'custom' / 'CMOR_coordinates.dat')
assert (
custom_table._cmor_folder ==
str(Path(root).parent / 'tables' / 'custom')
)
assert custom_table._user_table_folder == str(tmp_path)

# Make sure that default tables have been read
assert 'alb' in custom_table.tables['custom']
assert 'latitude' in custom_table.coords

# Make sure that custom tables have been read
assert 'newvarfortesting' in custom_table.tables['custom']
assert 'newcoordfortesting' in custom_table.coords
netcre = custom_table.get_variable('custom', 'netcre')
assert netcre.standard_name == 'air_temperature'
assert netcre.units == 'K'
assert netcre.long_name == 'This is New'

cmip6_table = CMOR_TABLES['CMIP6']
assert cmip6_table.default is custom_table
Expand Down
Loading

0 comments on commit e03d320

Please sign in to comment.