Remove old config-user.yml code

esmvalcore/_config/__init__.py

@@ -1,19 +1,15 @@
 """ESMValTool configuration."""
 from ._config import (
     get_activity,
+    get_extra_facets,
     get_institutes,
     get_project_config,
-    get_extra_facets,
     load_config_developer,
-    read_config_developer_file,
-    read_config_user_file,
 )
 from ._diagnostics import DIAGNOSTICS, TAGS
 from ._logging import configure_logging
 
 __all__ = (
-    'read_config_user_file',
-    'read_config_developer_file',
     'load_config_developer',
     'get_extra_facets',
     'get_project_config',

esmvalcore/_config/_config.py

@@ -1,17 +1,13 @@
 """Functions dealing with config-user.yml / config-developer.yml."""
 import collections.abc
-import datetime
 import fnmatch
 import logging
-import os
 import sys
-import warnings
 from functools import lru_cache
 from pathlib import Path
 
 import yaml
 
-from esmvalcore.cmor.check import CheckLevels
 from esmvalcore.cmor.table import CMOR_TABLES, read_cmor_tables
 from esmvalcore.exceptions import RecipeError
 
@@ -79,143 +75,18 @@ def pattern_filter(patterns, name):
         return [pat for pat in patterns if fnmatch.fnmatchcase(name, pat)]
 
     extra_facets = {}
-    for dataset_ in pattern_filter(project_details, dataset.facets['dataset']):
-        for mip_ in pattern_filter(project_details[dataset_], dataset.facets['mip']):
+    for dataset_ in pattern_filter(project_details, dataset['dataset']):
+        for mip_ in pattern_filter(project_details[dataset_], dataset['mip']):
             for var in pattern_filter(project_details[dataset_][mip_],
-                                      dataset.facets['short_name']):
+                                      dataset['short_name']):
                 facets = project_details[dataset_][mip_][var]
                 extra_facets.update(facets)
 
     return extra_facets
 
 
-def read_config_user_file(config_file, folder_name, options=None):
-    """Read config user file and store settings in a dictionary."""
-    if not config_file:
-        config_file = '~/.esmvaltool/config-user.yml'
-    config_file = _normalize_path(config_file)
-    # Read user config file
-    if not os.path.exists(config_file):
-        print(f"ERROR: Config file {config_file} does not exist")
-
-    with open(config_file, 'r') as file:
-        cfg = yaml.safe_load(file)
-
-    if options is None:
-        options = dict()
-    for key, value in options.items():
-        cfg[key] = value
-
-    # set defaults
-    defaults = {
-        'auxiliary_data_dir': '~/auxiliary_data',
-        'check_level': CheckLevels.DEFAULT,
-        'compress_netcdf': False,
-        'config_developer_file': None,
-        'drs': {},
-        'download_dir': '~/climate_data',
-        'exit_on_warning': False,
-        'extra_facets_dir': tuple(),
-        'max_parallel_tasks': None,
-        'max_years': None,
-        'offline': True,
-        'output_file_type': 'png',
-        'output_dir': '~/esmvaltool_output',
-        'profile_diagnostic': False,
-        'remove_preproc_dir': True,
-        'resume_from': [],
-        'run_diagnostic': True,
-        'save_intermediary_cubes': False,
-    }
-
-    for key in defaults:
-        if key not in cfg:
-            logger.info(
-                "No %s specification in config file, "
-                "defaulting to %s", key, defaults[key])
-            cfg[key] = defaults[key]
-
-    cfg['output_dir'] = _normalize_path(cfg['output_dir'])
-    cfg['download_dir'] = _normalize_path(cfg['download_dir'])
-    cfg['auxiliary_data_dir'] = _normalize_path(cfg['auxiliary_data_dir'])
-
-    if isinstance(cfg['extra_facets_dir'], str):
-        cfg['extra_facets_dir'] = (_normalize_path(cfg['extra_facets_dir']), )
-    else:
-        cfg['extra_facets_dir'] = tuple(
-            _normalize_path(p) for p in cfg['extra_facets_dir'])
-
-    cfg['config_developer_file'] = _normalize_path(
-        cfg['config_developer_file'])
-    cfg['config_file'] = config_file
-
-    for section in ['rootpath', 'drs']:
-        if 'obs4mips' in cfg[section]:
-            logger.warning(
-                "Correcting capitalization, project 'obs4mips'"
-                " should be written as 'obs4MIPs' in %s in %s", section,
-                config_file)
-            cfg[section]['obs4MIPs'] = cfg[section].pop('obs4mips')
-
-    for key in cfg['rootpath']:
-        root = cfg['rootpath'][key]
-        if isinstance(root, str):
-            cfg['rootpath'][key] = [_normalize_path(root)]
-        else:
-            cfg['rootpath'][key] = [_normalize_path(path) for path in root]
-
-    # insert a directory date_time_recipe_usertag in the output paths
-    now = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S")
-    new_subdir = '_'.join((folder_name, now))
-    cfg['output_dir'] = os.path.join(cfg['output_dir'], new_subdir)
-
-    # create subdirectories
-    cfg['preproc_dir'] = os.path.join(cfg['output_dir'], 'preproc')
-    cfg['work_dir'] = os.path.join(cfg['output_dir'], 'work')
-    cfg['plot_dir'] = os.path.join(cfg['output_dir'], 'plots')
-    cfg['run_dir'] = os.path.join(cfg['output_dir'], 'run')
-
-    # Read developer configuration file
-    load_config_developer(cfg['config_developer_file'])
-
-    # Validate configuration using the experimental module to avoid a crash
-    # after running the recipe because the html output writer uses this.
-    # In the long run, we need to replace this module with the Session from
-    # the experimental module.
-    with warnings.catch_warnings():
-        # ignore experimental API warning
-        warnings.simplefilter("ignore")
-        from esmvalcore.experimental.config._config_object import Session
-    Session.from_config_user(cfg)
-
-    return cfg
-
-
-def _normalize_path(path):
-    """Normalize paths.
-
-    Expand ~ character and environment variables and convert path to absolute.
-
-    Parameters
-    ----------
-    path: str
-        Original path
-
-    Returns
-    -------
-    str:
-        Normalized path
-    """
-    if path is None:
-        return None
-    return os.path.abspath(os.path.expanduser(os.path.expandvars(path)))
-
-
-def read_config_developer_file(cfg_file=None):
+def load_config_developer(cfg_file):
     """Read the developer's configuration file."""
-    if cfg_file is None:
-        cfg_file = Path(__file__).parents[1] / 'config-developer.yml'
-
     with open(cfg_file, 'r') as file:
         cfg = yaml.safe_load(file)
 
@@ -225,15 +96,9 @@ def read_config_developer_file(cfg_file=None):
             " should be written as 'obs4MIPs' in %s", cfg_file)
         cfg['obs4MIPs'] = cfg.pop('obs4mips')
 
-    return cfg
-
-
-def load_config_developer(cfg_file=None):
-    """Load the config developer file and initialize CMOR tables."""
-    cfg_developer = read_config_developer_file(cfg_file)
-    for key, value in cfg_developer.items():
+    for key, value in cfg.items():
         CFG[key] = value
-    read_cmor_tables(CFG)
+    read_cmor_tables(cfg_file)
 
 
 def get_project_config(project):

esmvalcore/_config/_esgf_pyclient.py

@@ -20,8 +20,6 @@
 
 import yaml
 
-from ._config import _normalize_path
-
 keyring: Optional[ModuleType] = None
 try:
     keyring = importlib.import_module('keyring')
@@ -169,7 +167,8 @@ def load_esgf_pyclient_config():
         cfg[section].update(file_cfg.get(section, {}))
 
     if 'cache' in cfg['search_connection']:
-        cache_file = _normalize_path(cfg['search_connection']['cache'])
+        cache_file = Path(os.path.expandvars(
+            cfg['search_connection']['cache'])).expanduser().absolute()
         cfg['search_connection']['cache'] = cache_file
         Path(cache_file).parent.mkdir(parents=True, exist_ok=True)
 

esmvalcore/_main.py

@@ -374,29 +374,10 @@ def run(self,
         from ._recipe import TASKSEP
         from .cmor.check import CheckLevels
         from .esgf._logon import logon
-        from .experimental import CFG
-
-        # Check validity of optional command line arguments with experimental
-        # API
         with warnings.catch_warnings():
             # ignore experimental API warning
             warnings.simplefilter("ignore")
-            from .experimental.config._config_object import Config as ExpConfig
-        explicit_optional_kwargs = {
-            'config_file': config_file,
-            'resume_from': resume_from,
-            'max_datasets': max_datasets,
-            'max_years': max_years,
-            'skip_nonexistent': skip_nonexistent,
-            'offline': offline,
-            'diagnostics': diagnostics,
-            'check_level': check_level,
-        }
-        all_optional_kwargs = dict(kwargs)
-        for (key, val) in explicit_optional_kwargs.items():
-            if val is not None:
-                all_optional_kwargs[key] = val
-        ExpConfig(all_optional_kwargs)
+            from .experimental import CFG
 
         recipe = self._get_recipe(recipe)
         CFG.load_from_file(config_file)

esmvalcore/cmor/table.py

@@ -10,7 +10,7 @@
 import logging
 import os
 from collections import Counter
-from functools import total_ordering
+from functools import lru_cache, total_ordering
 from pathlib import Path
 from typing import Dict, Type
 
@@ -75,25 +75,40 @@ def get_var_info(project, mip, short_name):
     return CMOR_TABLES[project].get_variable(mip, short_name)
 
 
-def read_cmor_tables(cfg_developer=None):
+def read_cmor_tables(cfg_file: Path):
     """Read cmor tables required in the configuration.
 
     Parameters
     ----------
-    cfg_developer : dict of str
-        Parsed config-developer file
+    cfg_file: pathlib.Path
+        Path to config-developer.yml file.
     """
-    if cfg_developer is None:
-        cfg_file = Path(__file__).parents[1] / 'config-developer.yml'
-        with cfg_file.open() as file:
-            cfg_developer = yaml.safe_load(file)
+    mtime = cfg_file.stat().st_mtime
+    cmor_tables = _read_cmor_tables(cfg_file, mtime)
+    CMOR_TABLES.clear()
+    CMOR_TABLES.update(cmor_tables)
+
+
+@lru_cache
+def _read_cmor_tables(cfg_file: Path, mtime: float):
+    """Read cmor tables required in the configuration.
 
+    Parameters
+    ----------
+    cfg_file: pathlib.Path
+        Path to config-developer.yml file.
+    mtime: float
+        Modification time of config-developer.yml file. Only used to
+        make sure the file is read again when it is changed.
+    """
+    with cfg_file.open('r', encoding='utf-8') as file:
+        cfg_developer = yaml.safe_load(file)
     cwd = os.path.dirname(os.path.realpath(__file__))
     var_alt_names_file = os.path.join(cwd, 'variable_alt_names.yml')
     with open(var_alt_names_file, 'r') as yfile:
         alt_names = yaml.safe_load(yfile)
 
-    CMOR_TABLES.clear()
+    cmor_tables = {}
 
     # Try to infer location for custom tables from config-developer.yml file,
     # if not possible, use default location
@@ -103,14 +118,15 @@ def read_cmor_tables(cfg_developer=None):
     if custom_path is not None:
         custom_path = os.path.expandvars(os.path.expanduser(custom_path))
     custom = CustomInfo(custom_path)
-    CMOR_TABLES['custom'] = custom
+    cmor_tables['custom'] = custom
 
     install_dir = os.path.dirname(os.path.realpath(__file__))
     for table in cfg_developer:
         if table == 'custom':
             continue
-        CMOR_TABLES[table] = _read_table(cfg_developer, table, install_dir,
+        cmor_tables[table] = _read_table(cfg_developer, table, install_dir,
                                          custom, alt_names)
+    return cmor_tables
 
 
 def _read_table(cfg_developer, table, install_dir, custom, alt_names):
@@ -957,4 +973,5 @@ def _read_table_file(self, table_file, table=None):
                     return
 
 
-read_cmor_tables()
+# Load the default tables on initializing the module.
+read_cmor_tables(Path(__file__).parents[1] / 'config-developer.yml')

esmvalcore/experimental/config/_config_object.py

@@ -198,42 +198,6 @@ def main_log_debug(self):
         """Return main log debug file."""
         return self.session_dir / self.relative_main_log_debug
 
-    def to_config_user(self) -> dict:
-        """Turn the `Session` object into a recipe-compatible dict.
-
-        This dict is compatible with the `config-user` argument in
-        :obj:`esmvalcore._recipe.Recipe`.
-        """
-        dct = self.copy()
-        dct['run_dir'] = self.run_dir
-        dct['work_dir'] = self.work_dir
-        dct['preproc_dir'] = self.preproc_dir
-        dct['plot_dir'] = self.plot_dir
-        dct['output_dir'] = self.session_dir
-        return dct
-
-    @classmethod
-    def from_config_user(cls, config_user: dict) -> 'Session':
-        """Convert `config-user` dict to API-compatible `Session` object.
-
-        For example, `_recipe.Recipe._cfg`.
-        """
-        dct = config_user.copy()
-        dct.pop('run_dir')
-        dct.pop('work_dir')
-        dct.pop('preproc_dir')
-        dct.pop('plot_dir')
-
-        session = cls(dct)
-
-        output_dir = Path(dct['output_dir']).parent
-        session_name = Path(dct['output_dir']).name
-
-        session['output_dir'] = output_dir
-        session.session_name = session_name
-
-        return session
-
 
 def _read_config_file(config_file):
     """Read config user file and store settings in a dictionary."""