diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index f3d1793b92..c14bcb6eab 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -699,11 +699,18 @@ Custom CMOR tables As mentioned in the previous section, the CMOR tables of projects that use ``cmor_strict: false`` will be extended with custom CMOR tables. -By default, these are loaded from `esmvalcore/cmor/tables/custom +For derived variables (the ones with ``derive: true`` in the recipe), the +custom CMOR tables will always be considered. +By default, these custom tables are loaded from `esmvalcore/cmor/tables/custom `_. However, by using the special project ``custom`` in the ``config-developer.yml`` file with the option ``cmor_path``, a custom location -for these custom CMOR tables can be specified: +for these custom CMOR tables can be specified. +In this case, the default custom tables are extended with those entries from +the custom location (in case of duplication, the custom location tables take +precedence). + +Example: .. code-block:: yaml @@ -743,11 +750,10 @@ Example for the file ``CMOR_asr.dat``: !---------------------------------- ! -It is also possible to use a special coordinates file ``CMOR_coordinates.dat``. -If this is not present in the custom directory, the one from the default -directory (`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat -`_) -is used. +It is also possible to use a special coordinates file ``CMOR_coordinates.dat``, +which will extend the entries from the default one +(`esmvalcore/cmor/tables/custom/CMOR_coordinates.dat +`_). .. _filterwarnings_config-developer: diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 3ee00696f4..6eba32426d 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -262,7 +262,7 @@ def get_variable( self, table_name: str, short_name: str, - derived: Optional[bool] = False, + derived: bool = False, ) -> VariableInfo | None: """Search and return the variable information. @@ -299,8 +299,8 @@ def get_variable( # cmor_strict=False or derived=True var_info = self._look_in_all_tables(derived, alt_names_list) - # If that didn' work either, look in default table if cmor_strict=False - # or derived=True + # If that didn't work either, look in default table if + # cmor_strict=False or derived=True if not var_info: var_info = self._look_in_default(derived, alt_names_list, table_name) @@ -988,51 +988,52 @@ class CustomInfo(CMIP5Info): Parameters ---------- - cmor_tables_path: str or None + cmor_tables_path: Full path to the table or name for the table if it is present in - ESMValTool repository - """ - - def __init__(self, cmor_tables_path=None): - cwd = os.path.dirname(os.path.realpath(__file__)) - default_cmor_folder = os.path.join(cwd, 'tables', 'custom') + ESMValTool repository. If ``None``, use default tables from + `esmvalcore/cmor/tables/custom`. - # Get custom location of CMOR tables if possible - if cmor_tables_path is None: - self._cmor_folder = default_cmor_folder - else: - self._cmor_folder = self._get_cmor_path(cmor_tables_path) - if not os.path.isdir(self._cmor_folder): - raise ValueError(f"Custom CMOR tables path {self._cmor_folder} is " - f"not a directory") + """ + def __init__(self, cmor_tables_path: Optional[str | Path] = None) -> None: + """Initialize class member.""" + self.coords = {} self.tables = {} - self.var_to_freq = {} + self.var_to_freq: dict[str, dict] = {} table = TableInfo() table.name = 'custom' self.tables[table.name] = table - # Try to read coordinates from custom location, use default location if - # not possible - coordinates_file = os.path.join( - self._cmor_folder, - 'CMOR_coordinates.dat', - ) - if os.path.isfile(coordinates_file): - self._coordinates_file = coordinates_file + # First, read default custom tables from repository + self._cmor_folder = self._get_cmor_path('custom') + self._read_table_dir(self._cmor_folder) + + # Second, if given, update default tables with user-defined custom + # tables + if cmor_tables_path is not None: + self._user_table_folder = self._get_cmor_path(cmor_tables_path) + if not os.path.isdir(self._user_table_folder): + raise ValueError( + f"Custom CMOR tables path {self._user_table_folder} is " + f"not a directory" + ) + self._read_table_dir(self._user_table_folder) else: - self._coordinates_file = os.path.join( - default_cmor_folder, - 'CMOR_coordinates.dat', - ) + self._user_table_folder = None - self.coords = {} - self._read_table_file(self._coordinates_file, self.tables['custom']) - for dat_file in glob.glob(os.path.join(self._cmor_folder, '*.dat')): - if dat_file == self._coordinates_file: + def _read_table_dir(self, table_dir: str) -> None: + """Read CMOR tables from directory.""" + # If present, read coordinates + coordinates_file = os.path.join(table_dir, 'CMOR_coordinates.dat') + if os.path.isfile(coordinates_file): + self._read_table_file(coordinates_file) + + # Read other variables + for dat_file in glob.glob(os.path.join(table_dir, '*.dat')): + if dat_file == coordinates_file: continue try: - self._read_table_file(dat_file, self.tables['custom']) + self._read_table_file(dat_file) except Exception: msg = f"Exception raised when loading {dat_file}" # Logger may not be ready at this stage @@ -1042,29 +1043,40 @@ def __init__(self, cmor_tables_path=None): print(msg) raise - def get_variable(self, table, short_name, derived=False): + def get_variable( + self, + table: str, + short_name: str, + derived: bool = False + ) -> VariableInfo | None: """Search and return the variable info. Parameters ---------- - table: str - Table name - short_name: str - Variable's short name - derived: bool, optional + table: + Table name. Ignored for custom tables. + short_name: + Variable's short name. + derived: Variable is derived. Info retrieval for derived variables always - look on the default tables if variable is not find in the - requested table + looks on the default tables if variable is not found in the + requested table. Ignored for custom tables. Returns ------- - VariableInfo - Return the VariableInfo object for the requested variable if - found, returns None if not + VariableInfo | None + `VariableInfo` object for the requested variable if found, returns + None if not. + """ return self.tables['custom'].get(short_name, None) - def _read_table_file(self, table_file, table=None): + def _read_table_file( + self, + table_file: str, + _: Optional[TableInfo] = None, + ) -> None: + """Read a single table file.""" with open(table_file, 'r', encoding='utf-8') as self._current_table: self._read_line() while True: @@ -1079,7 +1091,9 @@ def _read_table_file(self, table_file, table=None): self.coords[value] = self._read_coordinate(value) continue elif key == 'variable_entry': - table[value] = self._read_variable(value, '') + self.tables['custom'][value] = self._read_variable( + value, '' + ) continue if not self._read_line(): return diff --git a/tests/integration/cmor/test_read_cmor_tables.py b/tests/integration/cmor/test_read_cmor_tables.py index 32a472b316..a77b9b2946 100644 --- a/tests/integration/cmor/test_read_cmor_tables.py +++ b/tests/integration/cmor/test_read_cmor_tables.py @@ -1,4 +1,5 @@ from pathlib import Path +from textwrap import dedent import pytest import yaml @@ -7,17 +8,6 @@ from esmvalcore.cmor.table import __file__ as root from esmvalcore.cmor.table import read_cmor_tables -CUSTOM_CFG_DEVELOPER = { - 'custom': {'cmor_path': Path(root).parent / 'tables' / 'custom'}, - 'CMIP6': { - 'cmor_strict': True, - 'input_dir': {'default': '/'}, - 'input_file': '*.nc', - 'output_file': 'out.nc', - 'cmor_type': 'CMIP6', - }, -} - def test_read_cmor_tables_raiser(): """Test func raiser.""" @@ -52,12 +42,108 @@ def test_read_cmor_tables(): assert Path(table._cmor_folder) == table_path / 'obs4mips' / 'Tables' assert table.strict is False + project = 'custom' + table = CMOR_TABLES[project] + assert Path(table._cmor_folder) == table_path / 'custom' + assert table._user_table_folder is None + assert table.coords + assert table.tables['custom'] + + +CMOR_NEWVAR_ENTRY = dedent( + """ + !============ + variable_entry: newvarfortesting + !============ + modeling_realm: atmos + !---------------------------------- + ! Variable attributes: + !---------------------------------- + standard_name: + units: kg s m A + cell_methods: time: mean + cell_measures: area: areacella + long_name: Custom Variable for Testing + !---------------------------------- + ! Additional variable information: + !---------------------------------- + dimensions: longitude latitude time + type: real + positive: up + !---------------------------------- + ! + """ +) +CMOR_NETCRE_ENTRY = dedent( + """ + !============ + variable_entry: netcre + !============ + modeling_realm: atmos + !---------------------------------- + ! Variable attributes: + !---------------------------------- + standard_name: air_temperature ! for testing + units: K ! for testing + cell_methods: time: mean + cell_measures: area: areacella + long_name: This is New ! for testing + !---------------------------------- + ! Additional variable information: + !---------------------------------- + dimensions: longitude latitude time + type: real + positive: up + !---------------------------------- + ! + """ +) +CMOR_NEWCOORD_ENTRY = dedent( + """ + !============ + axis_entry: newcoordfortesting + !============ + !---------------------------------- + ! Axis attributes: + !---------------------------------- + standard_name: + units: kg + axis: Y ! X, Y, Z, T (default: undeclared) + long_name: Custom Coordinate for Testing + !---------------------------------- + ! Additional axis information: + !---------------------------------- + out_name: newcoordfortesting + valid_min: -90.0 + valid_max: 90.0 + stored_direction: increasing + type: double + must_have_bounds: yes + !---------------------------------- + ! + """ +) + def test_read_custom_cmor_tables(tmp_path): """Test reading of custom CMOR tables.""" + (tmp_path / 'CMOR_newvarfortesting.dat').write_text(CMOR_NEWVAR_ENTRY) + (tmp_path / 'CMOR_netcre.dat').write_text(CMOR_NETCRE_ENTRY) + (tmp_path / 'CMOR_coordinates.dat').write_text(CMOR_NEWCOORD_ENTRY) + + custom_cfg_developer = { + 'custom': {'cmor_path': str(tmp_path)}, + 'CMIP6': { + 'cmor_strict': True, + 'input_dir': {'default': '/'}, + 'input_file': '*.nc', + 'output_file': 'out.nc', + 'cmor_type': 'CMIP6', + }, + } cfg_file = tmp_path / 'config-developer.yml' with cfg_file.open('w', encoding='utf-8') as file: - yaml.safe_dump(CUSTOM_CFG_DEVELOPER, file) + yaml.safe_dump(custom_cfg_developer, file) read_cmor_tables(cfg_file) @@ -66,10 +152,23 @@ def test_read_custom_cmor_tables(tmp_path): assert 'custom' in CMOR_TABLES custom_table = CMOR_TABLES['custom'] - assert (Path(custom_table._cmor_folder) == - Path(root).parent / 'tables' / 'custom') - assert (Path(custom_table._coordinates_file) == - Path(root).parent / 'tables' / 'custom' / 'CMOR_coordinates.dat') + assert ( + custom_table._cmor_folder == + str(Path(root).parent / 'tables' / 'custom') + ) + assert custom_table._user_table_folder == str(tmp_path) + + # Make sure that default tables have been read + assert 'alb' in custom_table.tables['custom'] + assert 'latitude' in custom_table.coords + + # Make sure that custom tables have been read + assert 'newvarfortesting' in custom_table.tables['custom'] + assert 'newcoordfortesting' in custom_table.coords + netcre = custom_table.get_variable('custom', 'netcre') + assert netcre.standard_name == 'air_temperature' + assert netcre.units == 'K' + assert netcre.long_name == 'This is New' cmip6_table = CMOR_TABLES['CMIP6'] assert cmip6_table.default is custom_table diff --git a/tests/integration/cmor/test_table.py b/tests/integration/cmor/test_table.py index 793d0249e5..385869bc5f 100644 --- a/tests/integration/cmor/test_table.py +++ b/tests/integration/cmor/test_table.py @@ -416,32 +416,23 @@ def test_custom_tables_default_location(self): 'tables', 'custom', ) - expected_coordinate_file = os.path.join( - os.path.dirname(esmvalcore.cmor.__file__), - 'tables', - 'custom', - 'CMOR_coordinates.dat', - ) self.assertEqual(custom_info._cmor_folder, expected_cmor_folder) - self.assertEqual(custom_info._coordinates_file, - expected_coordinate_file) + self.assertTrue(custom_info.tables['custom']) + self.assertTrue(custom_info.coords) def test_custom_tables_location(self): """Test constructor with custom tables location.""" cmor_path = os.path.dirname(os.path.realpath(esmvalcore.cmor.__file__)) + default_cmor_tables_path = os.path.join(cmor_path, 'tables', 'custom') cmor_tables_path = os.path.join(cmor_path, 'tables', 'cmip5') cmor_tables_path = os.path.abspath(cmor_tables_path) + custom_info = CustomInfo(cmor_tables_path) - self.assertEqual(custom_info._cmor_folder, cmor_tables_path) - expected_coordinate_file = os.path.join( - os.path.dirname(esmvalcore.cmor.__file__), - 'tables', - 'custom', - 'CMOR_coordinates.dat', - ) - self.assertEqual(custom_info._coordinates_file, - expected_coordinate_file) + self.assertEqual(custom_info._cmor_folder, default_cmor_tables_path) + self.assertEqual(custom_info._user_table_folder, cmor_tables_path) + self.assertTrue(custom_info.tables['custom']) + self.assertTrue(custom_info.coords) def test_custom_tables_invalid_location(self): """Test constructor with invalid custom tables location."""