diff --git a/doc/recipe/overview.rst b/doc/recipe/overview.rst index 468acd1755..21e3b9e3e5 100644 --- a/doc/recipe/overview.rst +++ b/doc/recipe/overview.rst @@ -73,8 +73,8 @@ the following: Recipe section: ``datasets`` ============================ -The ``datasets`` section includes dictionaries that, via key-value pairs, define standardized -data specifications: +The ``datasets`` section includes dictionaries that, via key-value pairs or +"facets", define standardized data specifications: - dataset name (key ``dataset``, value e.g. ``MPI-ESM-LR`` or ``UKESM1-0-LL``). - project (key ``project``, value ``CMIP5`` or ``CMIP6`` for CMIP data, @@ -114,6 +114,162 @@ For example, a datasets section could be: - {dataset: HadGEM3-GC31-MM, project: CMIP6, exp: dcppA-hindcast, ensemble: r1i1p1f1, sub_experiment: s2000, grid: gn, start_year: 2000, end_year, 2002} - {dataset: BCC-CSM2-MR, project: CMIP6, exp: dcppA-hindcast, ensemble: r1i1p1f1, sub_experiment: s2000, grid: gn, timerange: '*'} +.. _dataset_wildcards: + +Automatically populating a recipe with all available datasets +------------------------------------------------------------- + +It is possible to use :obj:`glob` patterns or wildcards for certain facet +values, to make it easy to find all available datasets locally and/or on ESGF. +Note that ``project`` cannot be a wildcard. + +The facet values for local files are retrieved from the directory tree where the +directories represent the facets values. +Reading facet values from file names is not yet supported. +See :ref:`CMOR-DRS` for more information on this kind of file organization. + +When (some) files are available locally, the tool will not automatically look +for more files on ESGF. To populate a recipe with all available datasets from +ESGF, ``offline`` should be set to ``false`` and ``always_search_esgf`` should +be set to ``true`` in the +:ref:`user configuration file`. + +For more control over which datasets are selected, it is recommended to use +a Python script or `Jupyter notebook `_ to compose +the recipe. +See :ref:`/notebooks/composing-recipes.ipynb` for an example. +This is particularly useful when specific relations are required between +datasets, e.g. when a dataset needs to be available for multiple variables +or experiments. + +An example recipe that will use all CMIP6 datasets and all ensemble members +which have a ``'historical'`` experiment could look like this: + +.. code-block:: yaml + + datasets: + - project: CMIP6 + exp: historical + dataset: '*' + institute: '*' + ensemble: '*' + grid: '*' + +After running the recipe, a copy specifying exactly which datasets were used +is available in the output directory in the ``run`` subdirectory. +The filename of this recipe will end with ``_filled.yml``. + +For the ``timerange`` facet, special syntax is available. +See :ref:`timerange_examples` for more information. + +If populating a recipe using wildcards does not work, this is because there +were either no files found that match those facets, or the facets could not be +read from the directory name or ESGF. + +.. _supplementary_variables: + +Defining supplementary variables (ancillary variables and cell measures) +------------------------------------------------------------------------ + +It is common practice to store ancillary variables (e.g. land/sea/ice masks) +and cell measures (e.g. cell area, cell volume) in separate datasets that are +described by slightly different facets. +In ESMValCore, we call ancillary variables and cell measures "supplementary +variables". +Some :ref:`preprocessor functions ` need this information to +work. +For example, the :ref:`area_statistics` preprocessor function +needs to know area of each grid cell in order to compute a correctly weighted +statistic. + +To attach these variables to a dataset, the ``supplementary_variables`` keyword +can be used. +For example, to add cell area to a dataset, it can be specified as follows: + +.. code-block:: yaml + + datasets: + - dataset: BCC-ESM1 + project: CMIP6 + exp: historical + ensemble: r1i1p1f1 + grid: gn + supplementary_variables: + - short_name: areacella + mip: fx + exp: 1pctCO2 + +Note that the supplementary variable will inherit the facet values from the main +dataset, so only those facet values that differ need to be specified. + +.. _supplementary_dataset_wildcards: + +Automatically selecting the supplementary dataset +------------------------------------------------- + +When using many datasets, it may be quite a bit of work to find out which facet +values are required to find the corresponding supplementary data. +The tool can automatically guess the best matching supplementary dataset. +To use this feature, the supplementary dataset can be specified as: + +.. code-block:: yaml + + datasets: + - dataset: BCC-ESM1 + project: CMIP6 + exp: historical + ensemble: r1i1p1f1 + grid: gn + supplementary_variables: + - short_name: areacella + mip: fx + exp: '*' + activity: '*' + ensemble: '*' + +With this syntax, the tool will search all available values of ``exp``, +``activity``, and ``ensemble`` and use the supplementary dataset that shares the +most facet values with the main dataset. +Note that this behaviour is different from +:ref:`using wildcards in the main dataset `, +where they will be expanded to generate all matching datasets. +The available datasets are shown in the debug log messages when running a recipe +with wildcards, so if a different supplementary dataset is preferred, these +messages can be used to see what facet values are available. +The facet values for local files are retrieved from the directory tree where the +directories represent the facets values. +Reading facet values from file names is not yet supported. +If wildcard expansion fails, this is because there were either no files found +that match those facets, or the facets could not be read from the directory +name or ESGF. + +Automatic definition of supplementary variables +----------------------------------------------- + +If an ancillary variable or cell measure is +:ref:`needed by a preprocessor function `, +but it is not specified in the recipe, the tool will automatically make a best +guess using the syntax above. +Usually this will work fine, but if it does not, it is recommended to explicitly +define the supplementary variables in the recipe. + +To disable this automatic addition, define the supplementary variable as usual, +but add the special facet ``skip`` with value ``true``. +See :ref:`preprocessors_using_supplementary_variables` for an example recipe. + +Saving ancillary variables and cell measures +-------------------------------------------- + +By default, ancillary variables and cell measures will be removed +from the main variable before saving it to file because they can be as big as +the main variable. +To keep the supplementary variables, disable the preprocessor function that +removes them by setting ``remove_supplementary_variables: false`` in the +preprocessor profile in the recipe. + +Concatenating data corresponding to multiple facets +--------------------------------------------------- + It is possible to define the experiment as a list to concatenate two experiments. Here it is an example concatenating the `historical` experiment with `rcp85` @@ -130,6 +286,9 @@ In this case, the specified datasets are concatenated into a single cube: datasets: - {dataset: CanESM2, project: CMIP5, exp: [historical, rcp85], ensemble: [r1i1p1, r1i2p1], start_year: 2001, end_year: 2004} +Short notation of ensemble members and sub-experiments +------------------------------------------------------ + ESMValTool also supports a simplified syntax to add multiple ensemble members from the same dataset. In the ensemble key, any element in the form `(x:y)` will be replaced with all numbers from x to y (both inclusive), adding a dataset entry for each replacement. For example, to add ensemble members r1i1p1 to r10i1p1 @@ -152,7 +311,7 @@ Please, bear in mind that this syntax can only be used in the ensemble tag. Also, note that the combination of multiple experiments and ensembles, like exp: [historical, rcp85], ensemble: [r1i1p1, "r(2:3)i1p1"] is not supported and will raise an error. -The same simplified syntax can be used to add multiple sub-experiment ids: +The same simplified syntax can be used to add multiple sub-experiments: .. code-block:: yaml @@ -161,6 +320,9 @@ The same simplified syntax can be used to add multiple sub-experiment ids: .. _timerange_examples: +Time ranges +----------- + When using the ``timerange`` tag to specify the start and end points, possible values can be as follows: @@ -278,7 +440,7 @@ section will include: - a description of the diagnostic and lists of themes and realms that it applies to; - an optional ``additional_datasets`` section. - an optional ``title`` and ``description``, used to generate the title and description - of the ``index.html`` output file. + in the ``index.html`` output file. .. _tasks: @@ -286,9 +448,7 @@ The diagnostics section defines tasks ------------------------------------- The diagnostic section(s) define the tasks that will be executed when running the recipe. For each variable a preprocessing task will be defined and for each diagnostic script a -diagnostic task will be defined. If variables need to be derived -from other variables, a preprocessing task for each of the variables -needed to derive that variable will be defined as well. These tasks can be viewed +diagnostic task will be defined. These tasks can be viewed in the main_log_debug.txt file that is produced every run. Each task has a unique name that defines the subdirectory where the results of that task are stored. Task names start with the name of the diagnostic section followed by a '/' and then diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index ae421d6748..d4f142c03c 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -9,7 +9,7 @@ roughly following the default order in which preprocessor functions are applied: * :ref:`Variable derivation` * :ref:`CMOR check and dataset-specific fixes` -* :ref:`Fx variables as cell measures or ancillary variables` +* :ref:`preprocessors_using_supplementary_variables` * :ref:`Vertical interpolation` * :ref:`Weighting` * :ref:`Land/Sea/Ice masking` @@ -33,35 +33,7 @@ See :ref:`preprocessor_functions` for implementation details and the exact defau Overview ======== -.. - ESMValTool is a modular ``Python 3.8+`` software package possessing capabilities - of executing a large number of diagnostic routines that can be written in a - number of programming languages (Python, NCL, R, Julia). The modular nature - benefits the users and developers in different key areas: a new feature - developed specifically for version 2.0 is the preprocessing core or the - preprocessor (esmvalcore) that executes the bulk of standardized data - operations and is highly optimized for maximum performance in data-intensive - tasks. The main objective of the preprocessor is to integrate as many - standardizable data analysis functions as possible so that the diagnostics can - focus on the specific scientific tasks they carry. The preprocessor is linked - to the diagnostics library and the diagnostic execution is seamlessly performed - after the preprocessor has completed its steps. The benefit of having a - preprocessing unit separate from the diagnostics library include: - - * ease of integration of new preprocessing routines; - * ease of maintenance (including unit and integration testing) of existing - routines; - * a straightforward manner of importing and using the preprocessing routines as - part of the overall usage of the software and, as a special case, the use - during diagnostic execution; - * shifting the effort for the scientific diagnostic developer from implementing - both standard and diagnostic-specific functionalities to allowing them to - dedicate most of the effort to developing scientifically-relevant diagnostics - and metrics; - * a more strict code review process, given the smaller code base than for - diagnostics. - -The ESMValTool preprocessor can be used to perform a broad range of operations +The ESMValCore preprocessor can be used to perform a broad range of operations on the input data before diagnostics or metrics are applied. The preprocessor performs these operations in a centralized, documented and efficient way, thus reducing the data processing load on the diagnostics side. For an overview of @@ -88,7 +60,7 @@ The variable derivation module allows to derive variables which are not in the CMIP standard data request using standard variables as input. The typical use case of this operation is the evaluation of a variable which is only available in an observational dataset but not in the models. In this case a derivation -function is provided by the ESMValTool in order to calculate the variable and +function is provided by the ESMValCore in order to calculate the variable and perform the comparison. For example, several observational datasets deliver total column ozone as observed variable (`toz`), but CMIP models only provide the ozone 3D field. In this case, a derivation function is provided to @@ -129,8 +101,8 @@ CMORization and dataset-specific fixes Data checking ------------- -Data preprocessed by ESMValTool is automatically checked against its -cmor definition. To reduce the impact of this check while maintaining +Data preprocessed by ESMValCore is automatically checked against its +CMOR definition. To reduce the impact of this check while maintaining it as reliable as possible, it is split in two parts: one will check the metadata and will be done just after loading and concatenating the data and the other one will check the data itself and will be applied @@ -160,7 +132,7 @@ Dataset specific fixes ---------------------- Sometimes, the checker will detect errors that it can not fix by itself. -ESMValTool deals with those issues by applying specific fixes for those +ESMValCore deals with those issues by applying specific fixes for those datasets that require them. Fixes are applied at three different preprocessor steps: @@ -180,12 +152,180 @@ steps: To get an overview on data fixes and how to implement new ones, please go to :ref:`fixing_data`. -.. _Fx variables as cell measures or ancillary variables: +.. _preprocessors_using_supplementary_variables: + +Supplementary variables (ancillary variables and cell measures) +=============================================================== +The following preprocessor functions either require or prefer using an +`ancillary variable `_ +or +`cell measure `_ +to perform their computations. +In ESMValCore we call both types of variables "supplementary variables". + +============================================================== ============================== ===================================== +Preprocessor Variable short name Variable standard name +============================================================== ============================== ===================================== +:ref:`area_statistics` ``areacella``, ``areacello`` cell_area +:ref:`mask_landsea` ``sftlf``, ``sftof`` land_area_fraction, sea_area_fraction +:ref:`mask_landseaice` ``sftgif`` land_ice_area_fraction +:ref:`volume_statistics` ``volcello`` ocean_volume +:ref:`weighting_landsea_fraction` ``sftlf``, ``sftof`` land_area_fraction, sea_area_fraction +============================================================== ============================== ===================================== + +Only one of the listed variables is required. Supplementary variables can be +defined in the recipe as described in :ref:`supplementary_variables`. +In some cases, preprocessor functions may work without supplementary variables, +this is documented case by case in the preprocessor function definition. +If a preprocessor function requiring supplementary variables is used +without specifying these in the recipe, these will be automatically +added. +If the automatic selection does not give the desired result, specify the +supplementary variables in the recipe as described in +:ref:`supplementary_variables`. + +By default, supplementary variables will be removed from the +variable before saving it to file because they can be as big as the main +variable. +To keep the supplementary variables, disable the preprocessor +function :func:`esmvalcore.preprocessor.remove_supplementary_variables` that +removes them by setting ``remove_supplementary_variables: false`` in the +preprocessor in the recipe. + +Examples +-------- + +Compute the global mean surface air temperature, while +:ref:`automatically selecting the best matching supplementary dataset `: -Fx variables as cell measures or ancillary variables -==================================================== -The following preprocessors may require the use of ``fx_variables`` to be able -to perform the computations: +.. code-block:: yaml + + datasets: + - dataset: BCC-ESM1 + project: CMIP6 + ensemble: r1i1p1f1 + grid: gn + - dataset: MPI-ESM-MR + project: CMIP5 + ensemble: r1i1p1, + + preprocessors: + global_mean: + area_statistics: + operator: mean + + diagnostics: + example_diagnostic: + description: Global mean temperature. + variables: + tas: + mip: Amon + preprocessor: global_mean + exp: historical + timerange: '1990/2000' + supplementary_variables: + - short_name: areacella + mip: fx + exp: '*' + activity: '*' + ensemble: '*' + scripts: null + +Attach the land area fraction as an ancillary variable to surface air +temperature and store both in the same file: + +.. code-block:: yaml + + datasets: + - dataset: BCC-ESM1 + ensemble: r1i1p1f1 + grid: gn + + preprocessors: + keep_land_area_fraction: + remove_supplementary_variables: false + + diagnostics: + example_diagnostic: + description: Attach land area fraction. + variables: + tas: + mip: Amon + project: CMIP6 + preprocessor: keep_land_area_fraction + exp: historical + timerange: '1990/2000' + supplementary_variables: + - short_name: sftlf + mip: fx + exp: 1pctCO2 + scripts: null + + +Automatically define the required ancillary variable (``sftlf`` in this case) +and cell measure (``areacella``), but do not use ``areacella`` for dataset +``BCC-ESM1``: + +.. code-block:: yaml + + datasets: + - dataset: BCC-ESM1 + project: CMIP6 + ensemble: r1i1p1f1 + grid: gn + supplementary_variables: + - short_name: areacella + skip: true + - dataset: MPI-ESM-MR + project: CMIP5 + ensemble: r1i1p1 + + preprocessors: + global_land_mean: + mask_landsea: + mask_out: sea + area_statistics: + operator: mean + + diagnostics: + example_diagnostic: + description: Global mean temperature. + variables: + tas: + mip: Amon + preprocessor: global_land_mean + exp: historical + timerange: '1990/2000' + scripts: null + + +.. _`Fx variables as cell measures or ancillary variables`: + +Legacy method of specifying supplementary variables +--------------------------------------------------- + +.. deprecated:: 2.8.0 + The legacy method of specifying supplementary variables is deprecated and will + be removed in version 2.10.0. + To upgrade, remove all occurrences of ``fx_variables`` from your recipes and + rely on automatically defining the supplementary variables based on the + requirement of the preprocessor functions or specify them using the methods + described above. + To keep using the legacy behaviour until v2.10.0, set + ``use_legacy_supplementaries: true`` in the :ref:`user configuration file` or + run the tool with the flag ``--use-legacy-supplementaries=True``. + +Prior to version 2.8.0 of the tool, the supplementary variables could not be +defined at the variable or dataset level in the recipe, but could only be +defined in the preprocessor function that uses them using the ``fx_variables`` +argument. +This does not work well because in practice different datasets store their +supplementary variables under different facets. +For example, one dataset might only provide the ``areacella`` variable under the +``1pctCO2`` experiment while another one might only provide it for the +``historical`` experiment. +This forced the user to define a preprocessor per dataset, which was +inconvenient. ============================================================== ===================== Preprocessor Default fx variables @@ -239,7 +379,7 @@ or as a list of dictionaries: The recipe parser will automatically find the data files that are associated with these variables and pass them to the function for loading and processing. -If ``mip`` is not given, ESMValTool will search for the fx variable in all +If ``mip`` is not given, ESMValCore will search for the fx variable in all available tables of the specified project. .. warning:: @@ -270,7 +410,7 @@ that the defined preprocessor chain is applied to both ``variables`` and Note that when calling steps that require ``fx_variables`` inside diagnostic scripts, the variables are expected to contain the required ``cell_measures`` or -``ancillary_variables``. If missing, they can be added using the following functions: +``Fx variables as cell measures or ancillary variables``. If missing, they can be added using the following functions: .. code-block:: @@ -295,7 +435,7 @@ allows the scientist to perform a number of metrics specific to certain levels (whether it be air pressure or depth, e.g. the Quasi-Biennial-Oscillation (QBO) u30 is computed at 30 hPa). Dataset native vertical grids may not come with the desired set of levels, so an interpolation operation will be needed to regrid -the data vertically. ESMValTool can perform this vertical interpolation via the +the data vertically. ESMValCore can perform this vertical interpolation via the ``extract_levels`` preprocessor. Level extraction may be done in a number of ways. @@ -445,38 +585,14 @@ is for example useful for climate models which do not offer land/sea fraction files. This arguments also accepts the special dataset specifiers ``reference_dataset`` and ``alternative_dataset``. -Optionally you can specify your own custom fx variable to be used in cases when -e.g. a certain experiment is preferred for fx data retrieval: - -.. code-block:: yaml - - preprocessors: - preproc_weighting: - weighting_landsea_fraction: - area_type: land - exclude: ['CanESM2', 'reference_dataset'] - fx_variables: - sftlf: - exp: piControl - sftof: - exp: piControl - -or alternatively: +This function requires a land or sea area fraction `ancillary variable`_. +This supplementary variable, either ``sftlf`` or ``sftof``, should be attached +to the main dataset as described in :ref:`supplementary_variables`. -.. code-block:: yaml - - preprocessors: - preproc_weighting: - weighting_landsea_fraction: - area_type: land - exclude: ['CanESM2', 'reference_dataset'] - fx_variables: [ - {'short_name': 'sftlf', 'exp': 'piControl'}, - {'short_name': 'sftof', 'exp': 'piControl'} - ] - -More details on the argument ``fx_variables`` and its default values are given -in :ref:`Fx variables as cell measures or ancillary variables`. +.. deprecated:: 2.8.0 + The optional ``fx_variables`` argument specifies the fx variables that the + user wishes to input to the function. + More details on this are given in :ref:`Fx variables as cell measures or ancillary variables`. See also :func:`esmvalcore.preprocessor.weighting_landsea_fraction`. @@ -490,13 +606,13 @@ Introduction to masking ----------------------- Certain metrics and diagnostics need to be computed and performed on specific -domains on the globe. The ESMValTool preprocessor supports filtering +domains on the globe. The preprocessor supports filtering the input data on continents, oceans/seas and ice. This is achieved by masking the model data and keeping only the values associated with grid points that correspond to, e.g., land, ocean or ice surfaces, as specified by the user. Where possible, the masking is realized using the standard mask files provided together with the model data as part of the CMIP data request (the -so-called fx variable). In the absence of these files, the Natural Earth masks +so-called ancillary variable). In the absence of these files, the Natural Earth masks are used: although these are not model-specific, they represent a good approximation since they have a much higher resolution than most of the models and they are regularly updated with changing geographical features. @@ -506,11 +622,6 @@ and they are regularly updated with changing geographical features. Land-sea masking ---------------- -In ESMValTool, land-sea-ice masking can be done in two places: in the -preprocessor, to apply a mask on the data before any subsequent preprocessing -step and before running the diagnostic, or in the diagnostic scripts -themselves. We present both these implementations below. - To mask out a certain domain (e.g., sea) in the preprocessor, ``mask_landsea`` can be used: @@ -523,41 +634,19 @@ To mask out a certain domain (e.g., sea) in the preprocessor, and requires only one argument: ``mask_out``: either ``land`` or ``sea``. -Optionally you can specify your own custom fx variable to be used in cases when e.g. a certain -experiment is preferred for fx data retrieval. Note that it is possible to specify as many tags -for the fx variable as required: - - -.. code-block:: yaml - - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: - sftlf: - exp: piControl - sftof: - exp: piControl - ensemble: r2i1p1f1 - -or alternatively: +This function prefers using a land or sea area fraction `ancillary variable`_, +but if it is not available it will compute a mask based on +`Natural Earth `_ shapefiles. +This supplementary variable, either ``sftlf`` or ``sftof``, can be attached +to the main dataset as described in :ref:`supplementary_variables`. -.. code-block:: yaml - - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: [ - {'short_name': 'sftlf', 'exp': 'piControl'}, - {'short_name': 'sftof', 'exp': 'piControl', 'ensemble': 'r2i1p1f1'} - ] +.. deprecated:: 2.8.0 + The optional ``fx_variables`` argument specifies the fx variables that the + user wishes to input to the function. + More details on this are given in :ref:`Fx variables as cell measures or ancillary variables`. -More details on the argument ``fx_variables`` and its default values are given -in :ref:`Fx variables as cell measures or ancillary variables`. -If the corresponding fx file is not found (which is +If the corresponding ancillary variable is not available (which is the case for some models and almost all observational datasets), the preprocessor attempts to mask the data using Natural Earth mask files (that are vectorized rasters). As mentioned above, the spatial resolution of the the @@ -571,7 +660,7 @@ See also :func:`esmvalcore.preprocessor.mask_landsea`. Ice masking ----------- -Note that for masking out ice sheets, the preprocessor uses a different +For masking out ice sheets, the preprocessor uses a different function, to ensure that both land and sea or ice can be masked out without losing generality. To mask ice out, ``mask_landseaice`` can be used: @@ -584,32 +673,14 @@ losing generality. To mask ice out, ``mask_landseaice`` can be used: and requires only one argument: ``mask_out``: either ``landsea`` or ``ice``. -Optionally you can specify your own custom fx variable to be used in cases when -e.g. a certain experiment is preferred for fx data retrieval: - - -.. code-block:: yaml - - preprocessors: - landseaicemask: - mask_landseaice: - mask_out: sea - fx_variables: - sftgif: - exp: piControl - -or alternatively: +This function requires a land ice area fraction `ancillary variable`_. +This supplementary variable ``sftgif`` should be attached to the main dataset as +described in :ref:`supplementary_variables`. -.. code-block:: yaml - - preprocessors: - landseaicemask: - mask_landseaice: - mask_out: sea - fx_variables: [{'short_name': 'sftgif', 'exp': 'piControl'}] - -More details on the argument ``fx_variables`` and its default values are given -in :ref:`Fx variables as cell measures or ancillary variables`. +.. deprecated:: 2.8.0 + The optional ``fx_variables`` argument specifies the fx variables that the + user wishes to input to the function. + More details on this are given in :ref:`Fx variables as cell measures or ancillary variables`. See also :func:`esmvalcore.preprocessor.mask_landseaice`. @@ -639,7 +710,7 @@ Missing (masked) values can be a nuisance especially when dealing with multi-model ensembles and having to compute multi-model statistics; different numbers of missing data from dataset to dataset may introduce biases and artificially assign more weight to the datasets that have less missing data. -This is handled in ESMValTool via the missing values masks: two types of such +This is handled via the missing values masks: two types of such masks are available, one for the multi-model case and another for the single model case. @@ -713,7 +784,7 @@ regridding is based on the horizontal grid of another cube (the reference grid). If the horizontal grids of a cube and its reference grid are sufficiently the same, regridding is automatically and silently skipped for performance reasons. -The underlying regridding mechanism in ESMValTool uses +The underlying regridding mechanism in ESMValCore uses :obj:`iris.cube.Cube.regrid` from Iris. @@ -821,7 +892,7 @@ The arguments are defined below: Regridding (interpolation, extrapolation) schemes ------------------------------------------------- -ESMValTool has a number of built-in regridding schemes, which are presented in +ESMValCore has a number of built-in regridding schemes, which are presented in :ref:`built-in regridding schemes`. Additionally, it is also possible to use third party regridding schemes designed for use with :doc:`Iris `. This is explained in :ref:`generic regridding schemes`. @@ -914,7 +985,7 @@ One package that aims to capitalize on the :ref:`support for unstructured meshes introduced in Iris 3.2 ` is :doc:`iris-esmf-regrid:index`. It aims to provide lazy regridding for structured regular and irregular grids, as well as unstructured meshes. An -example of its usage in an ESMValTool preprocessor is: +example of its usage in a preprocessor is: .. code-block:: yaml @@ -1012,7 +1083,7 @@ Computing multi-model statistics is an integral part of model analysis and evaluation: individual models display a variety of biases depending on model set-up, initial conditions, forcings and implementation; comparing model data to observational data, these biases have a significantly lower statistical impact -when using a multi-model ensemble. ESMValTool has the capability of computing a +when using a multi-model ensemble. ESMValCore has the capability of computing a number of multi-model statistical measures: using the preprocessor module ``multi_model_statistics`` will enable the user to ask for either a multi-model ``mean``, ``median``, ``max``, ``min``, ``std_dev``, and / or ``pXX.YY`` with a set @@ -1774,9 +1845,17 @@ Note that this function is applied over the entire dataset. If only a specific region, depth layer or time period is required, then those regions need to be removed using other preprocessor operations in advance. -The optional ``fx_variables`` argument specifies the fx variables that the user -wishes to input to the function. More details on this are given in :ref:`Fx -variables as cell measures or ancillary variables`. +This function requires a cell area `cell measure`_, unless the coordinates of the +input data are regular 1D latitude and longitude coordinates so the cell areas +can be computed. +The required supplementary variable, either ``areacella`` for atmospheric variables +or ``areacello`` for ocean variables, can be attached to the main dataset +as described in :ref:`supplementary_variables`. + +.. deprecated:: 2.8.0 + The optional ``fx_variables`` argument specifies the fx variables that the user + wishes to input to the function. More details on this are given in :ref:`Fx + variables as cell measures or ancillary variables`. See also :func:`esmvalcore.preprocessor.area_statistics`. @@ -1821,11 +1900,19 @@ but maintains the time dimension. This function takes the argument: ``operator``, which defines the operation to apply over the volume. -No depth coordinate is required as this is determined by Iris. This function -works best when the ``fx_variables`` provide the cell volume. The optional -``fx_variables`` argument specifies the fx variables that the user wishes to -input to the function. More details on this are given in :ref:`Fx variables as -cell measures or ancillary variables`. +This function requires a cell volume `cell measure`_, unless the coordinates of +the input data are regular 1D latitude and longitude coordinates so the cell +volumes can be computed. +The required supplementary variable ``volcello`` can be attached to the main dataset +as described in :ref:`supplementary_variables`. + +No depth coordinate is required as this is determined by Iris. + +.. deprecated:: 2.8.0 + The optional ``fx_variables`` argument specifies the fx variables that the + user wishes to input to the function. + More details on this are given in + :ref:`Fx variables as cell measures or ancillary variables`. See also :func:`esmvalcore.preprocessor.volume_statistics`. @@ -1963,7 +2050,7 @@ See also :func:`esmvalcore.preprocessor.linear_trend_stderr`. Detrend ======= -ESMValTool also supports detrending along any dimension using +ESMValCore also supports detrending along any dimension using the preprocessor function 'detrend'. This function has two parameters: diff --git a/esmvalcore/_main.py b/esmvalcore/_main.py index 4eaad2e413..fb304deb2b 100755 --- a/esmvalcore/_main.py +++ b/esmvalcore/_main.py @@ -72,7 +72,6 @@ def process_recipe(recipe_file: Path, session): """Process recipe.""" import datetime import shutil - import warnings from esmvalcore._recipe.recipe import read_recipe_file if not recipe_file.is_file(): @@ -117,11 +116,7 @@ def process_recipe(recipe_file: Path, session): shutil.copy2(recipe_file, session.run_dir) # parse recipe - with warnings.catch_warnings(): - # ignore deprecation warning - warnings.simplefilter("ignore") - config_user = session.to_config_user() - recipe = read_recipe_file(recipe_file, config_user) + recipe = read_recipe_file(recipe_file, session) logger.debug("Recipe summary:\n%s", recipe) # run recipe.run() diff --git a/esmvalcore/_provenance.py b/esmvalcore/_provenance.py index 448560adee..dfe70e7220 100644 --- a/esmvalcore/_provenance.py +++ b/esmvalcore/_provenance.py @@ -105,7 +105,7 @@ class TrackedFile: def __init__(self, filename, - attributes, + attributes=None, ancestors=None, prov_filename=None): """Create an instance of a file with provenance tracking. @@ -115,7 +115,8 @@ def __init__(self, filename: str Path to the file on disk. attributes: dict - Dictionary with facets describing the file. + Dictionary with facets describing the file. If set to None, this + will be read from the file when provenance is initialized. ancestors: :obj:`list` of :obj:`TrackedFile` Ancestor files. prov_filename: str @@ -203,6 +204,12 @@ def _initialize_activity(self, activity): def _initialize_entity(self): """Initialize the entity representing the file.""" + if self.attributes is None: + self.attributes = {} + with Dataset(self.filename, 'r') as dataset: + for attr in dataset.ncattrs(): + self.attributes[attr] = dataset.getncattr(attr) + attributes = { 'attribute:' + str(k).replace(' ', '_'): str(v) for k, v in self.attributes.items() diff --git a/esmvalcore/_recipe/_io.py b/esmvalcore/_recipe/_io.py new file mode 100644 index 0000000000..937793a5fd --- /dev/null +++ b/esmvalcore/_recipe/_io.py @@ -0,0 +1,42 @@ +"""Functions for reading recipes.""" +from __future__ import annotations + +import os.path +from pathlib import Path +from typing import Any + +import yaml + + +def _copy(item): + """Create copies of mutable objects. + + This avoids accidental changes when a recipe contains the same + mutable object in multiple places due to the use of YAML anchors. + """ + if isinstance(item, dict): + new = {k: _copy(v) for k, v in item.items()} + elif isinstance(item, list): + new = [_copy(v) for v in item] + else: + new = item + return new + + +def _load_recipe(recipe: Path | str | dict[str, Any] | None) -> dict[str, Any]: + """Load a recipe from a file, string, dict, or create a new recipe.""" + if recipe is None: + recipe = { + 'diagnostics': {}, + } + + if isinstance(recipe, Path) or (isinstance(recipe, str) + and os.path.exists(recipe)): + recipe = Path(recipe).read_text(encoding='utf-8') + + if isinstance(recipe, str): + recipe = yaml.safe_load(recipe) + + recipe = _copy(recipe) + + return recipe # type: ignore diff --git a/esmvalcore/_recipe/check.py b/esmvalcore/_recipe/check.py index 68c00011dd..466561695d 100644 --- a/esmvalcore/_recipe/check.py +++ b/esmvalcore/_recipe/check.py @@ -1,10 +1,13 @@ """Module with functions to check a recipe.""" +from __future__ import annotations + import logging import os import re import subprocess from pprint import pformat from shutil import which +from typing import Any, Iterable import isodate import yamale @@ -13,6 +16,9 @@ from esmvalcore.local import _get_start_end_year, _parse_period from esmvalcore.preprocessor import TIME_PREPROCESSORS, PreprocessingTask from esmvalcore.preprocessor._multimodel import STATISTIC_MAPPING +from esmvalcore.preprocessor._supplementary_vars import ( + PREPROCESSOR_SUPPLEMENTARIES, +) logger = logging.getLogger(__name__) @@ -73,32 +79,43 @@ def diagnostics(diags): script_name, name)) -def duplicate_datasets(datasets): +def duplicate_datasets( + datasets: list[dict[str, Any]], + diagnostic: str, + variable_group: str, +) -> None: """Check for duplicate datasets.""" + if not datasets: + raise RecipeError( + "You have not specified any dataset or additional_dataset groups " + f"for variable {variable_group} in diagnostic {diagnostic}.") checked_datasets_ = [] for dataset in datasets: if dataset in checked_datasets_: raise RecipeError( - "Duplicate dataset {} in datasets section".format(dataset)) + f"Duplicate dataset {dataset} for variable {variable_group} " + f"in diagnostic {diagnostic}.") checked_datasets_.append(dataset) -def variable(var, required_keys): +def variable(var: dict[str, Any], required_keys: Iterable[str]): """Check variables as derived from recipe.""" required = set(required_keys) missing = required - set(var) if missing: raise RecipeError( - "Missing keys {} from variable {} in diagnostic {}".format( - missing, var.get('short_name'), var.get('diagnostic'))) + f"Missing keys {missing} in\n" + f"{pformat(var)}\n" + "for variable {var['variable_group']} in diagnostic " + f"{var['diagnostic']}") -def _log_data_availability_errors(input_files, var, patterns): +def _log_data_availability_errors(dataset): """Check if the required input data is available.""" - var = dict(var) + input_files = dataset.files + patterns = dataset._file_globs if not input_files: - var.pop('filename', None) - logger.error("No input files found for variable %s", var) + logger.error("No input files found for %s", dataset) if patterns: if len(patterns) == 1: msg = f': {patterns[0]}' @@ -134,20 +151,21 @@ def _group_years(years): return ", ".join(ranges) -def data_availability(input_files, var, patterns, log=True): +def data_availability(dataset, log=True): """Check if input_files cover the required years.""" + input_files = dataset.files + facets = dataset.facets + if log: - _log_data_availability_errors(input_files, var, patterns) + _log_data_availability_errors(dataset) if not input_files: - raise InputFilesNotFound( - f"Missing data for {var.get('alias', 'dataset')}: " - f"{var['short_name']}") + raise InputFilesNotFound(f"Missing data for {dataset.summary(True)}") - if 'timerange' not in var: + if 'timerange' not in facets: return - start_date, end_date = _parse_period(var['timerange']) + start_date, end_date = _parse_period(facets['timerange']) start_year = int(start_date[0:4]) end_year = int(end_date[0:4]) required_years = set(range(start_year, end_year + 1, 1)) @@ -166,6 +184,33 @@ def data_availability(input_files, var, patterns, log=True): missing_txt, "\n".join(str(f) for f in input_files))) +def preprocessor_supplementaries(dataset, settings): + """Check that the required supplementary variables have been added.""" + steps = [step for step in settings if step in PREPROCESSOR_SUPPLEMENTARIES] + supplementaries = {d.facets['short_name'] for d in dataset.supplementaries} + + for step in steps: + ancs = PREPROCESSOR_SUPPLEMENTARIES[step] + for short_name in ancs['variables']: + if short_name in supplementaries: + break + else: + if ancs['required'] == "require_at_least_one": + raise RecipeError( + f"Preprocessor function {step} requires that at least " + f"one supplementary variable of {ancs['variables']} is " + f"defined in the recipe for {dataset}.") + if ancs['required'] == "prefer_at_least_one": + logger.warning( + "Preprocessor function %s works best when at least " + "one supplementary variable of %s is defined in the " + "recipe for %s.", + step, + ancs['variables'], + dataset, + ) + + def tasks_valid(tasks): """Check that tasks are consistent.""" filenames = set() @@ -376,6 +421,15 @@ def valid_time_selection(timerange): _check_timerange_values(date, timerange) +def differing_timeranges(timeranges, required_vars): + """Log error if required variables have differing timeranges.""" + if len(timeranges) > 1: + raise ValueError( + f"Differing timeranges with values {timeranges} " + f"found for required variables {required_vars}. " + "Set `timerange` to a common value.") + + def reference_for_bias_preproc(products): """Check that exactly one reference dataset for bias preproc is given.""" step = 'bias' diff --git a/esmvalcore/_recipe/from_datasets.py b/esmvalcore/_recipe/from_datasets.py index 5bfab6f96a..145c2e95d0 100644 --- a/esmvalcore/_recipe/from_datasets.py +++ b/esmvalcore/_recipe/from_datasets.py @@ -4,14 +4,16 @@ import itertools import logging import re -from copy import deepcopy from functools import partial +from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, Iterable, Mapping, Sequence from nested_lookup import nested_delete from esmvalcore.exceptions import RecipeError +from ._io import _load_recipe + if TYPE_CHECKING: from esmvalcore.dataset import Dataset @@ -40,14 +42,14 @@ def _datasets_to_raw_recipe(datasets: Iterable[Dataset]) -> Recipe: facets.pop('diagnostic', None) if facets['short_name'] == variable_group: facets.pop('short_name') - if dataset.ancillaries: - facets['ancillary_variables'] = [] - for ancillary in dataset.ancillaries: + if dataset.supplementaries: + facets['supplementary_variables'] = [] + for supplementary in dataset.supplementaries: anc_facets = {} - for key, value in ancillary.minimal_facets.items(): + for key, value in supplementary.minimal_facets.items(): if facets.get(key) != value: anc_facets[key] = value - facets['ancillary_variables'].append(anc_facets) + facets['supplementary_variables'].append(anc_facets) variables[variable_group]['additional_datasets'].append(facets) recipe = {'diagnostics': diagnostics} @@ -95,18 +97,27 @@ def _move_datasets_up(recipe: Recipe) -> Recipe: return recipe +def _to_frozen(item): + """Return a frozen and sorted copy of nested dicts and lists.""" + if isinstance(item, list): + return tuple(sorted(_to_frozen(elem) for elem in item)) + if isinstance(item, dict): + return tuple(sorted((k, _to_frozen(v)) for k, v in item.items())) + return item + + def _move_one_level_up(base: dict, level: str, target: str): """Move datasets one level up in the recipe.""" groups = base[level] + if not groups: + return # Create a mapping from objects that can be hashed to the dicts # describing the datasets. dataset_mapping = {} for name, group in groups.items(): dataset_mapping[name] = { - tuple( - sorted((k, tuple(v) if isinstance(v, list) else v) - for k, v in ds.items())): ds + _to_frozen(ds): ds for ds in group['additional_datasets'] } @@ -266,9 +277,34 @@ def grouper(i, ens): return sorted(ensembles) # type: ignore +def _clean_recipe(recipe: Recipe, diagnostics: list[str]) -> Recipe: + """Clean up the input recipe.""" + # Format description nicer + if 'documentation' in recipe: + doc = recipe['documentation'] + for key in ['title', 'description']: + if key in doc: + doc[key] = doc[key].strip() + + # Filter out unused diagnostics + recipe['diagnostics'] = { + k: v + for k, v in recipe['diagnostics'].items() if k in diagnostics + } + + # Remove legacy supplementary definitions form the recipe + nested_delete( + recipe.get('preprocessors', {}), + 'fx_variables', + in_place=True, + ) + + return recipe + + def datasets_to_recipe( datasets: Iterable[Dataset], - recipe: dict | None = None, + recipe: Path | str | dict[str, Any] | None = None, ) -> dict: """Create or update a recipe from datasets. @@ -277,9 +313,10 @@ def datasets_to_recipe( datasets Datasets to use in the recipe. recipe - If provided, the datasets in the recipe will be replaced. The value - provided here should be a :ref:`recipe ` file that is loaded - using e.g. :func:`yaml.safe_load`. + :ref:`Recipe ` to load the datasets from. The value + provided here should be either a path to a file, a recipe file + that has been loaded using e.g. :func:`yaml.safe_load`, or an + :obj:`str` that can be loaded using :func:`yaml.safe_load`. Examples -------- @@ -296,20 +333,15 @@ def datasets_to_recipe( RecipeError Raised when a dataset is missing the ``diagnostic`` facet. """ - # TODO: should recipe be a dict, a string, or a file? - if recipe is None: - recipe = { - 'diagnostics': {}, - } - else: - recipe = deepcopy(recipe) + recipe = _load_recipe(recipe) + dataset_recipe = _datasets_to_recipe(datasets) + _clean_recipe(recipe, diagnostics=dataset_recipe['diagnostics']) # Remove dataset sections from recipe recipe.pop('datasets', None) nested_delete(recipe, 'additional_datasets', in_place=True) # Update datasets section - dataset_recipe = _datasets_to_recipe(datasets) if 'datasets' in dataset_recipe: recipe['datasets'] = dataset_recipe['datasets'] @@ -325,10 +357,4 @@ def datasets_to_recipe( if 'variables' in dataset_diagnostic: diagnostic['variables'] = dataset_diagnostic['variables'] - # Format description nicer - if 'documentation' in recipe: - doc = recipe['documentation'] - if 'description' in doc: - doc['description'] = doc['description'].strip() - return recipe diff --git a/esmvalcore/_recipe/recipe.py b/esmvalcore/_recipe/recipe.py index 702b4fd930..baac585026 100644 --- a/esmvalcore/_recipe/recipe.py +++ b/esmvalcore/_recipe/recipe.py @@ -1,40 +1,38 @@ """Recipe parser.""" +from __future__ import annotations + import fnmatch import logging import os -import re import warnings from collections import defaultdict from copy import deepcopy +from itertools import groupby from pathlib import Path from pprint import pformat +from typing import Any, Dict, Iterable, Sequence import yaml -from nested_lookup import get_all_keys, nested_delete, nested_lookup -from netCDF4 import Dataset from esmvalcore import __version__, esgf -from esmvalcore._provenance import TrackedFile, get_recipe_provenance +from esmvalcore._provenance import get_recipe_provenance from esmvalcore._task import DiagnosticTask, ResumeTask, TaskSet -from esmvalcore.cmor.check import CheckLevels -from esmvalcore.cmor.table import _CMOR_KEYS, CMOR_TABLES, _update_cmor_facets -from esmvalcore.config._config import ( - get_activity, - get_extra_facets, - get_institutes, - get_project_config, -) +from esmvalcore.cmor.table import CMOR_TABLES, _update_cmor_facets +from esmvalcore.config import CFG +from esmvalcore.config._config import TASKSEP, get_project_config from esmvalcore.config._diagnostics import TAGS -from esmvalcore.exceptions import InputFilesNotFound, RecipeError -from esmvalcore.local import _dates_to_timerange as dates_to_timerange -from esmvalcore.local import _get_multiproduct_filename -from esmvalcore.local import _get_output_file as get_output_file -from esmvalcore.local import _get_start_end_date as get_start_end_date +from esmvalcore.dataset import Dataset +from esmvalcore.exceptions import ( + ESMValCoreDeprecationWarning, + InputFilesNotFound, + RecipeError, +) from esmvalcore.local import ( - _get_timerange_from_years, + _dates_to_timerange, + _get_multiproduct_filename, + _get_output_file, _parse_period, _truncate_dates, - find_files, ) from esmvalcore.preprocessor import ( DEFAULT_ORDER, @@ -44,8 +42,6 @@ PreprocessingTask, PreprocessorFile, ) -from esmvalcore.preprocessor._derive import get_required -from esmvalcore.preprocessor._io import DATASET_KEYS, concatenate_callback from esmvalcore.preprocessor._other import _group_products from esmvalcore.preprocessor._regrid import ( _spec_to_latlonvals, @@ -53,73 +49,67 @@ get_reference_levels, parse_cell_spec, ) +from esmvalcore.preprocessor._supplementary_vars import ( + PREPROCESSOR_SUPPLEMENTARIES, +) +from esmvalcore.typing import Facets from . import check +from .from_datasets import datasets_to_recipe +from .to_datasets import ( + _derive_needed, + _get_input_datasets, + _representative_dataset, +) logger = logging.getLogger(__name__) -TASKSEP = os.sep +PreprocessorSettings = Dict[str, Any] DOWNLOAD_FILES = set() """Use a global variable to keep track of files that need to be downloaded.""" +USED_DATASETS = [] +"""Use a global variable to keep track of datasets that are actually used.""" + -def read_recipe_file(filename, config_user, initialize_tasks=True): +def read_recipe_file(filename: Path, session): """Read a recipe from file.""" check.recipe_with_schema(filename) - with open(filename, 'r') as file: + with open(filename, 'r', encoding='utf-8') as file: raw_recipe = yaml.safe_load(file) - return Recipe(raw_recipe, - config_user, - initialize_tasks, - recipe_file=filename) + return Recipe(raw_recipe, session, recipe_file=filename) -def _add_cmor_info(variable, override=False): - """Add information from CMOR tables to variable.""" - _update_cmor_facets(variable, override) - - # Check that keys are available - check.variable(variable, required_keys=_CMOR_KEYS) - - -def _add_extra_facets(variable, extra_facets_dir): - """Add extra_facets to variable.""" - extra_facets = get_extra_facets(variable["project"], variable["dataset"], - variable["mip"], variable["short_name"], - extra_facets_dir) - _augment(variable, extra_facets) - - -def _special_name_to_dataset(variable, special_name): +def _special_name_to_dataset(facets, special_name): """Convert special names to dataset names.""" if special_name in ('reference_dataset', 'alternative_dataset'): - if special_name not in variable: + if special_name not in facets: raise RecipeError( - "Preprocessor {preproc} uses {name}, but {name} is not " - "defined for variable {short_name} of diagnostic " - "{diagnostic}".format( - preproc=variable['preprocessor'], + "Preprocessor '{preproc}' uses '{name}', but '{name}' is not " + "defined for variable '{variable_group}' of diagnostic " + "'{diagnostic}'.".format( + preproc=facets['preprocessor'], name=special_name, - short_name=variable['short_name'], - diagnostic=variable['diagnostic'], + variable_group=facets['variable_group'], + diagnostic=facets['diagnostic'], )) - special_name = variable[special_name] + special_name = facets[special_name] return special_name -def _update_target_levels(variable, variables, settings, config_user): +def _update_target_levels(dataset, datasets, settings): """Replace the target levels dataset name with a filename if needed.""" levels = settings.get('extract_levels', {}).get('levels') if not levels: return - levels = _special_name_to_dataset(variable, levels) + levels = _special_name_to_dataset(dataset.facets, levels) # If levels is a dataset name, replace it by a dict with a 'dataset' entry - if any(levels == v['dataset'] for v in variables): + if any(levels == d.facets['dataset'] for d in datasets): settings['extract_levels']['levels'] = {'dataset': levels} levels = settings['extract_levels']['levels'] @@ -130,37 +120,32 @@ def _update_target_levels(variable, variables, settings, config_user): settings['extract_levels']['levels'] = get_cmor_levels( levels['cmor_table'], levels['coordinate']) elif 'dataset' in levels: - dataset = levels['dataset'] - if variable['dataset'] == dataset: + dataset_name = levels['dataset'] + if dataset.facets['dataset'] == dataset_name: del settings['extract_levels'] else: - variable_data = _get_dataset_info(dataset, variables) - filename = _dataset_to_file(variable_data, config_user) - fix_dir = f"{os.path.splitext(variable_data['filename'])[0]}_fixed" + target_ds = _select_dataset(dataset_name, datasets) + representative_ds = _representative_dataset(target_ds) + check.data_availability(representative_ds) settings['extract_levels']['levels'] = get_reference_levels( - filename=filename, - project=variable_data['project'], - dataset=dataset, - short_name=variable_data['short_name'], - mip=variable_data['mip'], - frequency=variable_data['frequency'], - fix_dir=fix_dir, - ) + representative_ds) -def _update_target_grid(variable, variables, settings, config_user): +def _update_target_grid(dataset, datasets, settings): """Replace the target grid dataset name with a filename if needed.""" grid = settings.get('regrid', {}).get('target_grid') if not grid: return - grid = _special_name_to_dataset(variable, grid) + grid = _special_name_to_dataset(dataset.facets, grid) - if variable['dataset'] == grid: + if dataset.facets['dataset'] == grid: del settings['regrid'] - elif any(grid == v['dataset'] for v in variables): - settings['regrid']['target_grid'] = _dataset_to_file( - _get_dataset_info(grid, variables), config_user) + elif any(grid == d.facets['dataset'] for d in datasets): + representative_ds = _representative_dataset( + _select_dataset(grid, datasets)) + check.data_availability(representative_ds) + settings['regrid']['target_grid'] = representative_ds else: # Check that MxN grid spec is correct target_grid = settings['regrid']['target_grid'] @@ -171,452 +156,325 @@ def _update_target_grid(variable, variables, settings, config_user): _spec_to_latlonvals(**target_grid) -def _update_regrid_time(variable, settings): +def _update_regrid_time(dataset, settings): """Input data frequency automatically for regrid_time preprocessor.""" regrid_time = settings.get('regrid_time') if regrid_time is None: return frequency = settings.get('regrid_time', {}).get('frequency') if not frequency: - settings['regrid_time']['frequency'] = variable['frequency'] - - -def _get_dataset_info(dataset, variables): - for var in variables: - if var['dataset'] == dataset: - return var - raise RecipeError("Unable to find matching file for dataset" - "{}".format(dataset)) - - -def _augment(base, update): - """Update dict base with values from dict update.""" - for key in update: - if key not in base: - base[key] = update[key] - - -def _dataset_to_file(variable, config_user): - """Find the first file belonging to dataset from variable info.""" - (files, globs) = _get_input_files(variable, config_user) - if not files and variable.get('derive'): - required_vars = get_required(variable['short_name'], - variable['project']) - for required_var in required_vars: - _augment(required_var, variable) - _add_cmor_info(required_var, override=True) - _add_extra_facets(required_var, config_user['extra_facets_dir']) - (files, globs) = _get_input_files(required_var, config_user) - if files: - variable = required_var - break - check.data_availability(files, variable, globs) - return files[0] - - -def _limit_datasets(variables, profile, max_datasets=0): + settings['regrid_time']['frequency'] = dataset.facets['frequency'] + + +def _select_dataset(dataset_name, datasets): + for dataset in datasets: + if dataset.facets['dataset'] == dataset_name: + return dataset + diagnostic = datasets[0].facets['diagnostic'] + variable_group = datasets[0].facets['variable_group'] + raise RecipeError( + f"Unable to find dataset '{dataset_name}' in the list of datasets" + f"for variable '{variable_group}' of diagnostic '{diagnostic}'.") + + +def _limit_datasets(datasets, profile): """Try to limit the number of datasets to max_datasets.""" + max_datasets = datasets[0].session['max_datasets'] if not max_datasets: - return variables + return datasets logger.info("Limiting the number of datasets to %s", max_datasets) required_datasets = [ (profile.get('extract_levels') or {}).get('levels'), (profile.get('regrid') or {}).get('target_grid'), - variables[0].get('reference_dataset'), - variables[0].get('alternative_dataset'), + datasets[0].facets.get('reference_dataset'), + datasets[0].facets.get('alternative_dataset'), ] - limited = [v for v in variables if v['dataset'] in required_datasets] - for variable in variables: + limited = [d for d in datasets if d.facets['dataset'] in required_datasets] + for dataset in datasets: if len(limited) >= max_datasets: break - if variable not in limited: - limited.append(variable) + if dataset not in limited: + limited.append(dataset) - logger.info("Only considering %s", ', '.join(v['alias'] for v in limited)) + logger.info("Only considering %s", + ', '.join(d.facets['alias'] for d in limited)) return limited -def _get_default_settings(variable, config_user, derive=False): +def _get_default_settings(dataset): """Get default preprocessor settings.""" + session = dataset.session + facets = dataset.facets + settings = {} - # Configure loading - settings['load'] = { - 'callback': concatenate_callback, - } - # Configure concatenation - settings['concatenate'] = {} - - # Configure fixes - fix = deepcopy(variable) - # File fixes - fix_dir = os.path.splitext(variable['filename'])[0] + '_fixed' - settings['fix_file'] = dict(fix) - settings['fix_file']['output_dir'] = fix_dir - # Cube fixes - fix['frequency'] = variable['frequency'] - fix['check_level'] = config_user.get('check_level', CheckLevels.DEFAULT) - settings['fix_metadata'] = dict(fix) - settings['fix_data'] = dict(fix) - - # Configure time extraction - if 'timerange' in variable and variable['frequency'] != 'fx': - settings['clip_timerange'] = {'timerange': variable['timerange']} - - if derive: + # Configure (deprecated, remove for v2.10.0) load callback + settings['load'] = {'callback': 'default'} + + if _derive_needed(dataset): settings['derive'] = { - 'short_name': variable['short_name'], - 'standard_name': variable['standard_name'], - 'long_name': variable['long_name'], - 'units': variable['units'], + 'short_name': facets['short_name'], + 'standard_name': facets['standard_name'], + 'long_name': facets['long_name'], + 'units': facets['units'], } - # Configure CMOR metadata check - settings['cmor_check_metadata'] = { - 'cmor_table': variable['project'], - 'mip': variable['mip'], - 'short_name': variable['short_name'], - 'frequency': variable['frequency'], - 'check_level': config_user.get('check_level', CheckLevels.DEFAULT) - } - # Configure final CMOR data check - settings['cmor_check_data'] = dict(settings['cmor_check_metadata']) - # Clean up fixed files - if not config_user['save_intermediary_cubes']: + if not session['save_intermediary_cubes']: + fix_dirs = [] + for item in [dataset] + dataset.supplementaries: + output_file = _get_output_file(item.facets, session.preproc_dir) + fix_dir = f"{output_file.with_suffix('')}_fixed" + fix_dirs.append(fix_dir) settings['cleanup'] = { - 'remove': [fix_dir], + 'remove': fix_dirs, } + # Strip supplementary variables before saving + settings['remove_supplementary_variables'] = {} + # Configure saving cubes to file - settings['save'] = {'compress': config_user['compress_netcdf']} - if variable['short_name'] != variable['original_short_name']: - settings['save']['alias'] = variable['short_name'] - - # Configure fx settings - settings['add_fx_variables'] = { - 'fx_variables': {}, - 'check_level': config_user.get('check_level', CheckLevels.DEFAULT) - } - settings['remove_fx_variables'] = {} + settings['save'] = {'compress': session['compress_netcdf']} + if facets['short_name'] != facets['original_short_name']: + settings['save']['alias'] = facets['short_name'] return settings -def _add_fxvar_keys(fx_info, variable, extra_facets_dir): - """Add keys specific to fx variable to use get_input_filelist.""" - fx_variable = deepcopy(variable) - fx_variable.update(fx_info) - fx_variable['variable_group'] = fx_info['short_name'] - - # add special ensemble for CMIP5 only - if fx_variable['project'] == 'CMIP5': - fx_variable['ensemble'] = 'r0i0p0' - - # add missing cmor info - _add_cmor_info(fx_variable, override=True) - - # add extra_facets - _add_extra_facets(fx_variable, extra_facets_dir) - - return fx_variable +def _guess_fx_mip(facets: dict, dataset: Dataset): + """Search mip for fx variable.""" + project = facets.get('project', dataset.facets['project']) + # check if project in config-developer + get_project_config(project) + tables = CMOR_TABLES[project].tables -def _search_fx_mip(tables, variable, fx_info, config_user): - """Search mip for fx variable.""" # Get all mips that offer that specific fx variable mips_with_fx_var = [] - for (mip, table) in tables.items(): - if fx_info['short_name'] in table: + for mip in tables: + if facets['short_name'] in tables[mip]: mips_with_fx_var.append(mip) # List is empty -> no table includes the fx variable if not mips_with_fx_var: raise RecipeError( - f"Requested fx variable '{fx_info['short_name']}' not available " - f"in any CMOR table for '{variable['project']}'") + f"Requested fx variable '{facets['short_name']}' not available " + f"in any CMOR table for '{project}'") # Iterate through all possible mips and check if files are available; in # case of ambiguity raise an error fx_files_for_mips = {} for mip in mips_with_fx_var: - fx_info['mip'] = mip - fx_info = _add_fxvar_keys(fx_info, variable, - config_user['extra_facets_dir']) logger.debug("For fx variable '%s', found table '%s'", - fx_info['short_name'], mip) - fx_files = _get_input_files(fx_info, config_user)[0] + facets['short_name'], mip) + fx_dataset = dataset.copy(**facets) + fx_dataset.supplementaries = [] + fx_dataset.set_facet('mip', mip) + fx_dataset.facets.pop('timerange', None) + fx_files = fx_dataset.files if fx_files: - logger.debug("Found fx variables '%s':\n%s", fx_info['short_name'], + logger.debug("Found fx variables '%s':\n%s", facets['short_name'], pformat(fx_files)) fx_files_for_mips[mip] = fx_files # Dict contains more than one element -> ambiguity if len(fx_files_for_mips) > 1: raise RecipeError( - f"Requested fx variable '{fx_info['short_name']}' for dataset " - f"'{variable['dataset']}' of project '{variable['project']}' is " - f"available in more than one CMOR table for " - f"'{variable['project']}': {sorted(list(fx_files_for_mips))}") + f"Requested fx variable '{facets['short_name']}' for dataset " + f"'{dataset.facets['dataset']}' of project '{project}' is " + f"available in more than one CMOR MIP table for " + f"'{project}': {sorted(fx_files_for_mips)}") # Dict is empty -> no files found -> handled at later stage if not fx_files_for_mips: - fx_info['mip'] = variable['mip'] - fx_files = [] + return mips_with_fx_var[0] # Dict contains one element -> ok - else: - mip = list(fx_files_for_mips)[0] - fx_info['mip'] = mip - fx_info = _add_fxvar_keys(fx_info, variable, - config_user['extra_facets_dir']) - fx_files = fx_files_for_mips[mip] - - return fx_info, fx_files + mip = list(fx_files_for_mips)[0] + return mip + + +def _set_default_preproc_fx_variables( + dataset: Dataset, + settings: PreprocessorSettings, +) -> None: + """Update `fx_variables` key in preprocessor settings with defaults.""" + default_fx = { + 'area_statistics': { + 'areacella': None, + }, + 'mask_landsea': { + 'sftlf': None, + }, + 'mask_landseaice': { + 'sftgif': None, + }, + 'volume_statistics': { + 'volcello': None, + }, + 'weighting_landsea_fraction': { + 'sftlf': None, + }, + } + if dataset.facets['project'] != 'obs4MIPs': + default_fx['area_statistics']['areacello'] = None + default_fx['mask_landsea']['sftof'] = None + default_fx['weighting_landsea_fraction']['sftof'] = None + + for step, fx_variables in default_fx.items(): + if step in settings and 'fx_variables' not in settings[step]: + settings[step]['fx_variables'] = fx_variables + + +def _get_supplementaries_from_fx_variables( + settings: PreprocessorSettings +) -> list[Facets]: + """Read supplementary facets from `fx_variables` in preprocessor.""" + supplementaries = [] + for step, kwargs in settings.items(): + allowed = PREPROCESSOR_SUPPLEMENTARIES.get(step, + {}).get('variables', []) + if fx_variables := kwargs.get('fx_variables'): + + if isinstance(fx_variables, list): + result: dict[str, Facets] = {} + for fx_variable in fx_variables: + if isinstance(fx_variable, str): + # Legacy legacy method of specifying fx variable + short_name = fx_variable + result[short_name] = {} + elif isinstance(fx_variable, dict): + short_name = fx_variable['short_name'] + result[short_name] = fx_variable + fx_variables = result + + for short_name, facets in fx_variables.items(): + if short_name not in allowed: + raise RecipeError( + f"Preprocessor function '{step}' does not support " + f"supplementary variable '{short_name}'") + if facets is None: + facets = {} + facets['short_name'] = short_name + supplementaries.append(facets) + + return supplementaries + + +def _get_legacy_supplementary_facets( + dataset: Dataset, + settings: PreprocessorSettings, +) -> list[Facets]: + """Load the supplementary dataset facets from the preprocessor settings.""" + # First update `fx_variables` in preprocessor settings with defaults + _set_default_preproc_fx_variables(dataset, settings) + + supplementaries = _get_supplementaries_from_fx_variables(settings) + + # Guess the ensemble and mip if they is not specified + for facets in supplementaries: + if 'ensemble' not in facets and dataset.facets['project'] == 'CMIP5': + facets['ensemble'] = 'r0i0p0' + if 'mip' not in facets: + facets['mip'] = _guess_fx_mip(facets, dataset) + return supplementaries + + +def _add_legacy_supplementary_datasets(dataset: Dataset, settings): + """Update fx settings depending on the needed method.""" + if not dataset.session['use_legacy_supplementaries']: + return + if dataset.supplementaries: + # Supplementaries have been defined in the recipe. + # Just remove any skipped supplementaries (they have been kept so we + # know that supplementaries have been defined in the recipe). + dataset.supplementaries = [ + ds for ds in dataset.supplementaries + if not ds.facets.get('skip', False) + ] + return + logger.debug("Using legacy method to add supplementaries to %s", dataset) -def _get_fx_files(variable, fx_info, config_user): - """Get fx files (searching all possible mips).""" - # assemble info from master variable - var_project = variable['project'] - # check if project in config-developer - try: - get_project_config(var_project) - except ValueError: - raise RecipeError(f"Requested fx variable '{fx_info['short_name']}' " - f"with parent variable '{variable}' does not have " - f"a '{var_project}' project in config-developer.") - project_tables = CMOR_TABLES[var_project].tables - - # If mip is not given, search all available tables. If the variable is not - # found or files are available in more than one table, raise error - if not fx_info['mip']: - fx_info, fx_files = _search_fx_mip(project_tables, variable, fx_info, - config_user) - else: - mip = fx_info['mip'] - if mip not in project_tables: - raise RecipeError( - f"Requested mip table '{mip}' for fx variable " - f"'{fx_info['short_name']}' not available for project " - f"'{var_project}'") - if fx_info['short_name'] not in project_tables[mip]: - raise RecipeError( - f"fx variable '{fx_info['short_name']}' not available in CMOR " - f"table '{mip}' for '{var_project}'") - fx_info = _add_fxvar_keys(fx_info, variable, - config_user['extra_facets_dir']) - fx_files = _get_input_files(fx_info, config_user)[0] + legacy_ds = dataset.copy() + for facets in _get_legacy_supplementary_facets(dataset, settings): + legacy_ds.add_supplementary(**facets) - # Flag a warning if no files are found - if not fx_files: - logger.warning("Missing data for fx variable '%s' of dataset %s", - fx_info['short_name'], - fx_info['alias'].replace('_', ' ')) + for supplementary_ds in legacy_ds.supplementaries: + _update_cmor_facets(supplementary_ds.facets, override=True) + if supplementary_ds.files: + dataset.supplementaries.append(supplementary_ds) - # If frequency = fx, only allow a single file - if fx_files: - if fx_info['frequency'] == 'fx': - fx_files = fx_files[0] + dataset._fix_fx_exp() - return fx_files, fx_info + # Remove preprocessor keyword argument `fx_variables` + for kwargs in settings.values(): + kwargs.pop('fx_variables', None) -def _exclude_dataset(settings, variable, step): +def _exclude_dataset(settings, facets, step): """Exclude dataset from specific preprocessor step if requested.""" exclude = { - _special_name_to_dataset(variable, dataset) + _special_name_to_dataset(facets, dataset) for dataset in settings[step].pop('exclude', []) } - if variable['dataset'] in exclude: + if facets['dataset'] in exclude: settings.pop(step) logger.debug("Excluded dataset '%s' from preprocessor step '%s'", - variable['dataset'], step) + facets['dataset'], step) -def _update_weighting_settings(settings, variable): +def _update_weighting_settings(settings, facets): """Update settings for the weighting preprocessors.""" if 'weighting_landsea_fraction' not in settings: return - _exclude_dataset(settings, variable, 'weighting_landsea_fraction') - - -def _update_fx_files(step_name, settings, variable, config_user, fx_vars): - """Update settings with mask fx file list or dict.""" - if not fx_vars: - return - for fx_var, fx_info in fx_vars.items(): - if not fx_info: - fx_info = {} - if 'mip' not in fx_info: - fx_info.update({'mip': None}) - if 'short_name' not in fx_info: - fx_info.update({'short_name': fx_var}) - fx_files, fx_info = _get_fx_files(variable, fx_info, config_user) - if fx_files: - fx_info['filename'] = fx_files - settings['add_fx_variables']['fx_variables'].update( - {fx_var: fx_info}) - logger.debug('Using fx files for variable %s during step %s: %s', - variable['short_name'], step_name, pformat(fx_files)) - - -def _fx_list_to_dict(fx_vars): - """Convert fx list to dictionary. - - To be deprecated at some point. - """ - user_fx_vars = {} - for fx_var in fx_vars: - if isinstance(fx_var, dict): - short_name = fx_var['short_name'] - user_fx_vars.update({short_name: fx_var}) - continue - user_fx_vars.update({fx_var: None}) - return user_fx_vars - - -def _update_fx_settings(settings, variable, config_user): - """Update fx settings depending on the needed method.""" - # Add default values to the option 'fx_variables' if it is not explicitly - # specified and transform fx variables to dicts - def _update_fx_vars_in_settings(step_settings, step_name): - """Update fx_variables option in the settings.""" - # Add default values for fx_variables - if 'fx_variables' not in step_settings: - default_fx = { - 'area_statistics': { - 'areacella': None, - }, - 'mask_landsea': { - 'sftlf': None, - }, - 'mask_landseaice': { - 'sftgif': None, - }, - 'volume_statistics': { - 'volcello': None, - }, - 'weighting_landsea_fraction': { - 'sftlf': None, - }, - } - if variable['project'] != 'obs4MIPs': - default_fx['area_statistics']['areacello'] = None - default_fx['mask_landsea']['sftof'] = None - default_fx['weighting_landsea_fraction']['sftof'] = None - step_settings['fx_variables'] = default_fx[step_name] - - # Transform fx variables to dicts - user_fx_vars = step_settings['fx_variables'] - if user_fx_vars is None: - step_settings['fx_variables'] = {} - elif isinstance(user_fx_vars, list): - step_settings['fx_variables'] = _fx_list_to_dict(user_fx_vars) - - fx_steps = [ - 'mask_landsea', 'mask_landseaice', 'weighting_landsea_fraction', - 'area_statistics', 'volume_statistics' - ] - for step_name in settings: - if step_name in fx_steps: - _update_fx_vars_in_settings(settings[step_name], step_name) - _update_fx_files(step_name, settings, variable, config_user, - settings[step_name]['fx_variables']) - # Remove unused attribute in 'fx_steps' preprocessors. - # The fx_variables information is saved in - # the 'add_fx_variables' step. - settings[step_name].pop('fx_variables', None) - - -def _read_attributes(filename): - """Read the attributes from a netcdf file.""" - attributes = {} - if not (os.path.exists(filename) - and os.path.splitext(filename)[1].lower() == '.nc'): - return attributes - - with Dataset(filename, 'r') as dataset: - for attr in dataset.ncattrs(): - attributes[attr] = dataset.getncattr(attr) - return attributes - - -def _get_input_files(variable, config_user): - """Get the input files for a single dataset (locally and via download).""" - if variable['frequency'] != 'fx': - start_year, end_year = _parse_period(variable['timerange']) - - start_year = int(str(start_year[0:4])) - end_year = int(str(end_year[0:4])) + _exclude_dataset(settings, facets, 'weighting_landsea_fraction') + + +def _add_to_download_list(dataset): + """Add the files of `dataset` to `DOWNLOAD_FILES`.""" + for i, file in enumerate(dataset.files): + if isinstance(file, esgf.ESGFFile): + DOWNLOAD_FILES.add(file) + dataset.files[i] = file.local_file(dataset.session['download_dir']) + + +def _schedule_for_download(datasets): + """Schedule files for download and show the list of files in the log.""" + for dataset in datasets: + _add_to_download_list(dataset) + for supplementary_ds in dataset.supplementaries: + _add_to_download_list(supplementary_ds) + + files = list(dataset.files) + for supplementary_ds in dataset.supplementaries: + files.extend(supplementary_ds.files) + + logger.debug( + "Using input files for variable %s of dataset %s:\n%s", + dataset.facets['short_name'], + dataset.facets['alias'].replace('_', ' '), + '\n'.join(f'{f} (will be downloaded)' if not f.exists() else str(f) + for f in files), + ) - variable['start_year'] = start_year - variable['end_year'] = end_year - variable = dict(variable) - if variable['project'] == 'CMIP5' and variable['frequency'] == 'fx': - variable['ensemble'] = 'r0i0p0' - if variable['frequency'] == 'fx': - variable.pop('timerange', None) - input_files, globs = find_files(debug=True, **variable) +def _check_input_files(input_datasets: Iterable[Dataset]) -> set[str]: + """Check that the required input files are available.""" + missing = set() - # Set up downloading from ESGF if requested. - if (not config_user['offline'] - and variable['project'] in esgf.facets.FACETS): - search_esgf = config_user['always_search_esgf'] - if not search_esgf: - # Only look on ESGF if files are not available locally. + for input_dataset in input_datasets: + for dataset in [input_dataset] + input_dataset.supplementaries: try: - check.data_availability( - input_files, - variable, - globs, - log=False, - ) - except RecipeError: - search_esgf = True - - if search_esgf: - local_files = set(Path(f).name for f in input_files) - search_result = esgf.find_files(**variable) - for file in search_result: - local_copy = file.local_file(config_user['download_dir']) - if local_copy.name not in local_files: - if not local_copy.exists(): - DOWNLOAD_FILES.add(file) - input_files.append(local_copy) - - globs.append('ESGF') - - return (input_files, globs) - - -def _get_ancestors(variable, config_user): - """Get the input files for a single dataset and setup provenance.""" - (input_files, globs) = _get_input_files(variable, config_user) - - logger.debug( - "Using input files for variable %s of dataset %s:\n%s", - variable['short_name'], - variable['alias'].replace('_', ' '), - '\n'.join( - f'{f} (will be downloaded)' if not os.path.exists(f) else str(f) - for f in input_files), - ) - check.data_availability(input_files, variable, globs) - logger.info("Found input files for %s", - variable['alias'].replace('_', ' ')) - - # Set up provenance tracking - for i, filename in enumerate(input_files): - attributes = _read_attributes(filename) - input_files[i] = TrackedFile(filename, attributes) + check.data_availability(dataset) + except RecipeError as exc: + missing.add(exc.message) - return input_files + return missing def _apply_preprocessor_profile(settings, profile_settings): @@ -652,7 +510,7 @@ def _get_common_attributes(products, settings): timerange = product.attributes['timerange'] start, end = _parse_period(timerange) if 'timerange' not in attributes: - attributes['timerange'] = dates_to_timerange(start, end) + attributes['timerange'] = _dates_to_timerange(start, end) else: start_date, end_date = _parse_period(attributes['timerange']) start_date, start = _truncate_dates(start_date, start) @@ -671,7 +529,7 @@ def _get_common_attributes(products, settings): start_date = min([start, start_date]) end_date = max([end, end_date]) - attributes['timerange'] = dates_to_timerange(start_date, end_date) + attributes['timerange'] = _dates_to_timerange(start_date, end_date) # Ensure that attributes start_year and end_year are always available start_year, end_year = _parse_period(attributes['timerange']) @@ -693,13 +551,13 @@ def _get_downstream_settings(step, order, products): return settings -def _update_multi_dataset_settings(variable, settings): +def _update_multi_dataset_settings(facets, settings): """Configure multi dataset statistics.""" for step in MULTI_MODEL_FUNCTIONS: if not settings.get(step): continue # Exclude dataset if requested - _exclude_dataset(settings, variable, step) + _exclude_dataset(settings, facets, step) def _update_warning_settings(settings, project): @@ -769,9 +627,11 @@ def _update_multiproduct(input_products, order, preproc_dir, step): statistic_attributes[step]) filename = _get_multiproduct_filename(statistic_attributes, preproc_dir) - statistic_attributes['filename'] = filename - statistic_product = PreprocessorFile(statistic_attributes, - downstream_settings) + statistic_product = PreprocessorFile( + filename=filename, + attributes=statistic_attributes, + settings=downstream_settings, + ) # Note that ancestors is set when running the preprocessor func. output_products.add(statistic_product) relevant_settings['output_products'][identifier][ statistic] = statistic_product @@ -788,169 +648,94 @@ def update_ancestors(ancestors, step, downstream_settings): settings[key] = value -def _update_extract_shape(settings, config_user): +def _update_extract_shape(settings, session): if 'extract_shape' in settings: shapefile = settings['extract_shape'].get('shapefile') if shapefile: if not os.path.exists(shapefile): shapefile = os.path.join( - config_user['auxiliary_data_dir'], + session['auxiliary_data_dir'], shapefile, ) settings['extract_shape']['shapefile'] = shapefile check.extract_shape(settings['extract_shape']) -def _update_timerange(variable, config_user): - """Update wildcards in timerange with found datetime values. - - If the timerange is given as a year, it ensures it's formatted as a - 4-digit value (YYYY). - """ - if 'timerange' not in variable: - return - - timerange = variable.get('timerange') - check.valid_time_selection(timerange) - - if '*' in timerange: - facets = deepcopy(variable) - facets.pop('timerange', None) - files = find_files(**facets) - if not files and not config_user.get('offline', True): - files = [file.name for file in esgf.find_files(**facets)] - - if not files: - raise InputFilesNotFound( - f"Missing data for {variable['alias']}: " - f"{variable['short_name']}. Cannot determine indeterminate " - f"time range '{timerange}'." - ) - - intervals = [get_start_end_date(name) for name in files] - - min_date = min(interval[0] for interval in intervals) - max_date = max(interval[1] for interval in intervals) - - if timerange == '*': - timerange = f'{min_date}/{max_date}' - if '*' in timerange.split('/')[0]: - timerange = timerange.replace('*', min_date) - if '*' in timerange.split('/')[1]: - timerange = timerange.replace('*', max_date) - - # Make sure that years are in format YYYY - (start_date, end_date) = timerange.split('/') - timerange = dates_to_timerange(start_date, end_date) - check.valid_time_selection(timerange) - - variable['timerange'] = timerange - - -def _match_products(products, variables): - """Match a list of input products to output product attributes.""" - grouped_products = defaultdict(list) - - if not products: - return grouped_products - - def get_matching(attributes): - """Find the output filename which matches input attributes best.""" - best_score = 0 - filenames = [] - for variable in variables: - filename = variable['filename'] - score = sum(v == variable.get(k) for k, v in attributes.items()) - - if score > best_score: - best_score = score - filenames = [filename] - elif score == best_score: - filenames.append(filename) - - if not filenames: - logger.warning( - "Unable to find matching output file for input file %s", - filename) - - return filenames - - # Group input files by output file - for product in products: - matching_filenames = get_matching(product.attributes) - for filename in matching_filenames: - grouped_products[filename].append(product) - - return grouped_products - - -def _allow_skipping(ancestors, variable, config_user): +def _allow_skipping(dataset: Dataset): """Allow skipping of datasets.""" allow_skipping = all([ - config_user.get('skip_nonexistent'), - not ancestors, - variable['dataset'] != variable.get('reference_dataset'), + dataset.session['skip_nonexistent'], + dataset.facets['dataset'] != dataset.facets.get('reference_dataset'), ]) return allow_skipping -def _get_preprocessor_products(variables, profile, order, ancestor_products, - config_user, name): +def _set_version(dataset: Dataset, input_datasets: list[Dataset]): + """Set the 'version' facet based on derivation input datasets.""" + versions = set() + for in_dataset in input_datasets: + in_dataset.set_version() + if version := in_dataset.facets.get('version'): + if isinstance(version, list): + versions.update(version) + else: + versions.add(version) + if versions: + version = versions.pop() if len(versions) == 1 else sorted(versions) + dataset.set_facet('version', version) + for supplementary_ds in dataset.supplementaries: + supplementary_ds.set_version() + + +def _get_preprocessor_products( + datasets: list[Dataset], + profile: dict[str, Any], + order: list[str], + name: str, +) -> set[PreprocessorFile]: """Get preprocessor product definitions for a set of datasets. It updates recipe settings as needed by various preprocessors and sets the correct ancestry. """ products = set() - preproc_dir = config_user['preproc_dir'] - for variable in variables: - if variable['frequency'] == 'fx': - variable.pop('timerange', None) - _update_timerange(variable, config_user) - variable['filename'] = get_output_file(variable, - config_user['preproc_dir']) + datasets = _limit_datasets(datasets, profile) - if ancestor_products: - grouped_ancestors = _match_products(ancestor_products, variables) - else: - grouped_ancestors = {} - - missing_vars = set() - for variable in variables: - settings = _get_default_settings( - variable, - config_user, - derive='derive' in profile, - ) - _update_warning_settings(settings, variable['project']) + missing_vars: set[str] = set() + for dataset in datasets: + dataset.augment_facets() + + for dataset in datasets: + settings = _get_default_settings(dataset) + _update_warning_settings(settings, dataset.facets['project']) _apply_preprocessor_profile(settings, profile) - _update_multi_dataset_settings(variable, settings) - try: - _update_target_levels( - variable=variable, - variables=variables, - settings=settings, - config_user=config_user, - ) - except RecipeError as ex: - missing_vars.add(ex.message) - _update_preproc_functions(settings, config_user, variable, variables, - missing_vars) - ancestors = grouped_ancestors.get(variable['filename']) - if not ancestors: - try: - ancestors = _get_ancestors(variable, config_user) - except RecipeError as ex: - if _allow_skipping(ancestors, variable, config_user): - logger.info("Skipping: %s", ex.message) - else: - missing_vars.add(ex.message) - continue + _update_multi_dataset_settings(dataset.facets, settings) + _update_preproc_functions(settings, dataset, datasets, missing_vars) + _add_legacy_supplementary_datasets(dataset, settings) + check.preprocessor_supplementaries(dataset, settings) + input_datasets = _get_input_datasets(dataset) + missing = _check_input_files(input_datasets) + if missing: + if _allow_skipping(dataset): + logger.info("Skipping: %s", missing) + else: + missing_vars.update(missing) + continue + _set_version(dataset, input_datasets) + USED_DATASETS.append(dataset) + _schedule_for_download(input_datasets) + logger.info("Found input files for %s", dataset.summary(shorten=True)) + + filename = _get_output_file( + dataset.facets, + dataset.session.preproc_dir, + ) product = PreprocessorFile( - attributes=variable, + filename=filename, + attributes=dataset.facets, settings=settings, - ancestors=ancestors, + datasets=input_datasets, ) products.add(product) @@ -963,6 +748,27 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, check.reference_for_bias_preproc(products) + _configure_multi_product_preprocessor( + products=products, + preproc_dir=datasets[0].session.preproc_dir, + profile=profile, + order=order, + ) + + for product in products: + _set_start_end_year(product) + product.check() + + return products + + +def _configure_multi_product_preprocessor( + products: Iterable[PreprocessorFile], + preproc_dir: Path, + profile: PreprocessorSettings, + order: Sequence[str], +): + """Configure preprocessing of ensemble and multimodel statistics.""" ensemble_step = 'ensemble_statistics' multi_model_step = 'multi_model_statistics' if ensemble_step in profile: @@ -1000,75 +806,79 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, else: multimodel_products = set() - for product in products | multimodel_products | ensemble_products: + for product in multimodel_products | ensemble_products: product.check() + _set_start_end_year(product) - # Ensure that attributes start_year and end_year are always available - # for all products if a timerange is specified - if 'timerange' in product.attributes: - start_year, end_year = _parse_period( - product.attributes['timerange']) - product.attributes['start_year'] = int(str(start_year[0:4])) - product.attributes['end_year'] = int(str(end_year[0:4])) - return products +def _set_start_end_year(product: PreprocessorFile) -> None: + """Set the attributes `start_year` and `end_year`. + + These attributes are used by many diagnostic scripts in ESMValTool. + """ + if 'timerange' in product.attributes: + start_year, end_year = _parse_period(product.attributes['timerange']) + product.attributes['start_year'] = int(str(start_year[0:4])) + product.attributes['end_year'] = int(str(end_year[0:4])) -def _update_preproc_functions(settings, config_user, variable, variables, - missing_vars): - _update_extract_shape(settings, config_user) - _update_weighting_settings(settings, variable) - _update_fx_settings(settings=settings, - variable=variable, - config_user=config_user) - _update_timerange(variable, config_user) +def _update_preproc_functions(settings, dataset, datasets, missing_vars): + session = dataset.session + _update_extract_shape(settings, session) + _update_weighting_settings(settings, dataset.facets) + try: + _update_target_levels( + dataset=dataset, + datasets=datasets, + settings=settings, + ) + except RecipeError as exc: + missing_vars.add(exc.message) try: _update_target_grid( - variable=variable, - variables=variables, + dataset=dataset, + datasets=datasets, settings=settings, - config_user=config_user, ) except RecipeError as ex: missing_vars.add(ex.message) - _update_regrid_time(variable, settings) + _update_regrid_time(dataset, settings) + if dataset.facets.get('frequency') == 'fx': + check.check_for_temporal_preprocs(settings) -def _get_single_preprocessor_task(variables, - profile, - config_user, - name, - ancestor_tasks=None): - """Create preprocessor tasks for a set of datasets w/ special case fx.""" - if ancestor_tasks is None: - ancestor_tasks = [] +def _get_preprocessor_task(datasets, profiles, task_name): + """Create preprocessor task(s) for a set of datasets.""" + # First set up the preprocessor profile + facets = datasets[0].facets + session = datasets[0].session + preprocessor = facets.get('preprocessor', 'default') + if preprocessor not in profiles: + raise RecipeError( + f"Unknown preprocessor '{preprocessor}' in variable " + f"{facets['variable_group']} of diagnostic {facets['diagnostic']}") + logger.info("Creating preprocessor '%s' task for variable '%s'", + preprocessor, facets['variable_group']) + profile = deepcopy(profiles[preprocessor]) order = _extract_preprocessor_order(profile) - ancestor_products = [p for task in ancestor_tasks for p in task.products] - - if variables[0].get('frequency') == 'fx': - check.check_for_temporal_preprocs(profile) - ancestor_products = None + # Create preprocessor task products = _get_preprocessor_products( - variables=variables, + datasets=datasets, profile=profile, order=order, - ancestor_products=ancestor_products, - config_user=config_user, - name=name, + name=task_name, ) if not products: - raise RecipeError( - "Did not find any input data for task {}".format(name)) + raise RecipeError(f"Did not find any input data for task {task_name}") task = PreprocessingTask( products=products, - ancestors=ancestor_tasks, - name=name, + name=task_name, order=order, - debug=config_user['save_intermediary_cubes'], - write_ncl_interface=config_user['write_ncl_interface'], + debug=session['save_intermediary_cubes'], + write_ncl_interface=session['write_ncl_interface'], ) logger.info("PreprocessingTask %s created.", task.name) @@ -1083,194 +893,85 @@ def _extract_preprocessor_order(profile): custom_order = profile.pop('custom_order', False) if not custom_order: return DEFAULT_ORDER - order = tuple(p for p in profile if p not in INITIAL_STEPS + FINAL_STEPS) - return INITIAL_STEPS + order + FINAL_STEPS - - -def _split_settings(settings, step, order=DEFAULT_ORDER): - """Split settings, using step as a separator.""" - before = {} - for _step in order: - if _step == step: - break - if _step in settings: - before[_step] = settings[_step] - after = { - k: v - for k, v in settings.items() if not (k == step or k in before) - } - return before, after - - -def _split_derive_profile(profile): - """Split the derive preprocessor profile.""" - order = _extract_preprocessor_order(profile) - before, after = _split_settings(profile, 'derive', order) - after['derive'] = True - after['fix_file'] = False - after['fix_metadata'] = False - after['fix_data'] = False - if order != DEFAULT_ORDER: - before['custom_order'] = True - after['custom_order'] = True - return before, after - - -def _check_differing_timeranges(timeranges, required_vars): - """Log error if required variables have differing timeranges.""" - if len(timeranges) > 1: - raise ValueError( - f"Differing timeranges with values {timeranges} " - f"found for required variables {required_vars}. " - "Set `timerange` to a common value.", - ) - - -def _get_derive_input_variables(variables, config_user): - """Determine the input sets of `variables` needed for deriving.""" - derive_input = {} - - def append(group_prefix, var): - """Append variable `var` to a derive input group.""" - group = group_prefix + var['short_name'] - var['variable_group'] = group - if group not in derive_input: - derive_input[group] = [] - derive_input[group].append(var) - - for variable in variables: - group_prefix = variable['variable_group'] + '_derive_input_' - if not variable.get('force_derivation') and \ - '*' in variable['timerange']: - raise RecipeError( - f"Error in derived variable: {variable['short_name']}: " - "Using 'force_derivation: false' (the default option) " - "in combination with wildcards ('*') in timerange is " - "not allowed; explicitly use 'force_derivation: true' " - "or avoid the use of wildcards in timerange." - ) - if not variable.get('force_derivation') and _get_input_files( - variable, config_user)[0]: - # No need to derive, just process normally up to derive step - var = deepcopy(variable) - append(group_prefix, var) - else: - # Process input data needed to derive variable - required_vars = get_required(variable['short_name'], - variable['project']) - timeranges = set() - for var in required_vars: - _augment(var, variable) - _add_cmor_info(var, override=True) - _add_extra_facets(var, config_user['extra_facets_dir']) - _update_timerange(var, config_user) - files = _get_input_files(var, config_user)[0] - if var.get('optional') and not files: - logger.info( - "Skipping: no data found for %s which is marked as " - "'optional'", var) - else: - append(group_prefix, var) - timeranges.add(var['timerange']) - _check_differing_timeranges(timeranges, required_vars) - variable['timerange'] = " ".join(timeranges) - - # An empty derive_input (due to all variables marked as 'optional' is - # handled at a later step - return derive_input - - -def _get_preprocessor_task(variables, profiles, config_user, task_name): - """Create preprocessor task(s) for a set of datasets.""" - # First set up the preprocessor profile - variable = variables[0] - preproc_name = variable.get('preprocessor') - if preproc_name not in profiles: - raise RecipeError( - "Unknown preprocessor {} in variable {} of diagnostic {}".format( - preproc_name, variable['short_name'], variable['diagnostic'])) - profile = deepcopy(profiles[variable['preprocessor']]) - logger.info("Creating preprocessor '%s' task for variable '%s'", - variable['preprocessor'], variable['short_name']) - variables = _limit_datasets(variables, profile, - config_user.get('max_datasets')) - for variable in variables: - _add_cmor_info(variable) - # Create preprocessor task(s) - derive_tasks = [] - # set up tasks - if variable.get('derive'): - # Create tasks to prepare the input data for the derive step - derive_profile, profile = _split_derive_profile(profile) - derive_input = _get_derive_input_variables(variables, config_user) - - for derive_variables in derive_input.values(): - for derive_variable in derive_variables: - _add_cmor_info(derive_variable, override=True) - derive_name = task_name.split( - TASKSEP)[0] + TASKSEP + derive_variables[0]['variable_group'] - task = _get_single_preprocessor_task( - derive_variables, - derive_profile, - config_user, - name=derive_name, - ) - derive_tasks.append(task) - - # Create (final) preprocessor task - task = _get_single_preprocessor_task( - variables, - profile, - config_user, - ancestor_tasks=derive_tasks, - name=task_name, - ) - - return task + if 'derive' not in profile: + initial_steps = INITIAL_STEPS + ('derive', ) + else: + initial_steps = INITIAL_STEPS + order = tuple(p for p in profile if p not in initial_steps + FINAL_STEPS) + return initial_steps + order + FINAL_STEPS class Recipe: """Recipe object.""" - info_keys = ('project', 'activity', 'driver', 'dataset', 'exp', - 'sub_experiment', 'ensemble', 'version') - """List of keys to be used to compose the alias, ordered by priority.""" - - def __init__(self, - raw_recipe, - config_user, - initialize_tasks=True, - recipe_file=None): + def __init__(self, raw_recipe, session, recipe_file: Path): """Parse a recipe file into an object.""" # Clear the global variable containing the set of files to download DOWNLOAD_FILES.clear() - self._download_files = set() - self._cfg = deepcopy(config_user) - self._cfg['write_ncl_interface'] = self._need_ncl( + USED_DATASETS.clear() + self._download_files: set[esgf.ESGFFile] = set() + self.session = session + self.session['write_ncl_interface'] = self._need_ncl( raw_recipe['diagnostics']) self._raw_recipe = raw_recipe - self._updated_recipe = {} - self._filename = os.path.basename(recipe_file) + self._filename = Path(recipe_file.name) self._preprocessors = raw_recipe.get('preprocessors', {}) if 'default' not in self._preprocessors: self._preprocessors['default'] = {} + self._set_use_legacy_supplementaries() + self.datasets = Dataset.from_recipe(recipe_file, session) self.diagnostics = self._initialize_diagnostics( - raw_recipe['diagnostics'], raw_recipe.get('datasets', [])) + raw_recipe['diagnostics']) self.entity = self._initialize_provenance( raw_recipe.get('documentation', {})) try: - self.tasks = self.initialize_tasks() if initialize_tasks else None + self.tasks = self.initialize_tasks() except RecipeError as exc: self._log_recipe_errors(exc) raise + def _set_use_legacy_supplementaries(self): + """Automatically determine if legacy supplementaries are used.""" + names = set() + steps = set() + for name, profile in self._preprocessors.items(): + for step, kwargs in profile.items(): + if isinstance(kwargs, dict) and 'fx_variables' in kwargs: + names.add(name) + steps.add(step) + if self.session['use_legacy_supplementaries'] is False: + kwargs.pop('fx_variables') + if names: + warnings.warn( + ESMValCoreDeprecationWarning( + "Encountered 'fx_variables' argument in preprocessor(s) " + f"{sorted(names)}, function(s) {sorted(steps)}. The " + "'fx_variables' argument is deprecated and will stop " + "working in v2.10. Please remove it and if automatic " + "definition of supplementary variables does not work " + "correctly, specify the supplementary variables in the " + "recipe as described in https://docs.esmvaltool.org/" + "projects/esmvalcore/en/latest/recipe/preprocessor.html" + "#ancillary-variables-and-cell-measures")) + if self.session['use_legacy_supplementaries'] is None: + logger.info("Running with --use-legacy-supplementaries=True") + self.session['use_legacy_supplementaries'] = True + + # Also set the global config because it is used to check if + # mismatching shapes should be ignored when attaching + # supplementary variables in `esmvalcore.preprocessor. + # _supplementary_vars.add_supplementary_variables` to avoid having to + # introduce a new function argument that is immediately deprecated. + option = 'use_legacy_supplementaries' + CFG[option] = self.session[option] + def _log_recipe_errors(self, exc): """Log a message with recipe errors.""" logger.error(exc.message) for task in exc.failed_tasks: logger.error(task.message) - if self._cfg['offline'] and any( + if self.session['offline'] and any( isinstance(err, InputFilesNotFound) for err in exc.failed_tasks): logger.error( @@ -1279,13 +980,13 @@ def _log_recipe_errors(self, exc): logger.error( "If the files are available locally, please check" " your `rootpath` and `drs` settings in your user " - "configuration file %s", self._cfg['config_file']) + "configuration file %s", self.session['config_file']) logger.error( "To automatically download the required files to " "`download_dir: %s`, set `offline: false` in %s or run the " "recipe with the extra command line argument --offline=False", - self._cfg['download_dir'], - self._cfg['config_file'], + self.session['download_dir'], + self.session['config_file'], ) logger.info( "Note that automatic download is only available for files" @@ -1316,7 +1017,7 @@ def _initialize_provenance(self, raw_documentation): return get_recipe_provenance(doc, self._filename) - def _initialize_diagnostics(self, raw_diagnostics, raw_datasets): + def _initialize_diagnostics(self, raw_diagnostics): """Define diagnostics in recipe.""" logger.debug("Retrieving diagnostics from recipe") check.diagnostics(raw_diagnostics) @@ -1326,13 +1027,9 @@ def _initialize_diagnostics(self, raw_diagnostics, raw_datasets): for name, raw_diagnostic in raw_diagnostics.items(): diagnostic = {} diagnostic['name'] = name - additional_datasets = raw_diagnostic.get('additional_datasets', []) - datasets = (raw_datasets + additional_datasets) - diagnostic['preprocessor_output'] = \ - self._initialize_preprocessor_output( - name, - raw_diagnostic.get('variables', {}), - datasets) + diagnostic['datasets'] = [ + ds for ds in self.datasets if ds.facets['diagnostic'] == name + ] variable_names = tuple(raw_diagnostic.get('variables', {})) diagnostic['scripts'] = self._initialize_scripts( name, raw_diagnostic.get('scripts'), variable_names) @@ -1344,241 +1041,6 @@ def _initialize_diagnostics(self, raw_diagnostics, raw_datasets): return diagnostics - @staticmethod - def _initialize_datasets(raw_datasets): - """Define datasets used by variable.""" - datasets = deepcopy(raw_datasets) - - for dataset in datasets: - for key in dataset: - DATASET_KEYS.add(key) - return datasets - - @staticmethod - def _expand_tag(variables, input_tag): - """Expand tags such as ensemble members or startdates. - - Expansion only supports ensembles defined as strings, not lists. - Returns the expanded datasets. - """ - expanded = [] - regex = re.compile(r'\(\d+:\d+\)') - - def expand_tag(variable, input_tag): - tag = variable.get(input_tag, "") - match = regex.search(tag) - if match: - start, end = match.group(0)[1:-1].split(':') - for i in range(int(start), int(end) + 1): - expand = deepcopy(variable) - expand[input_tag] = regex.sub(str(i), tag, 1) - expand_tag(expand, input_tag) - else: - expanded.append(variable) - - for variable in variables: - tag = variable.get(input_tag, "") - if isinstance(tag, (list, tuple)): - for elem in tag: - if regex.search(elem): - raise RecipeError( - f"In variable {variable}: {input_tag} expansion " - f"cannot be combined with {input_tag} lists") - expanded.append(variable) - else: - expand_tag(variable, input_tag) - - return expanded - - def _initialize_variables(self, raw_variable, raw_datasets): - """Define variables for all datasets.""" - variables = [] - - raw_variable = deepcopy(raw_variable) - datasets = self._initialize_datasets( - raw_datasets + raw_variable.pop('additional_datasets', [])) - if not datasets: - raise RecipeError("You have not specified any dataset " - "or additional_dataset groups " - f"for variable {raw_variable} Exiting.") - check.duplicate_datasets(datasets) - - for index, dataset in enumerate(datasets): - variable = deepcopy(raw_variable) - variable.update(dataset) - - variable['recipe_dataset_index'] = index - if 'end_year' in variable and self._cfg.get('max_years'): - variable['end_year'] = min( - variable['end_year'], - variable['start_year'] + self._cfg['max_years'] - 1) - variables.append(variable) - - required_keys = { - 'short_name', - 'mip', - 'dataset', - 'project', - 'preprocessor', - 'diagnostic', - } - if 'fx' not in raw_variable.get('mip', ''): - required_keys.update({'timerange'}) - else: - variable.pop('timerange', None) - for variable in variables: - _add_extra_facets(variable, self._cfg['extra_facets_dir']) - _get_timerange_from_years(variable) - if 'institute' not in variable: - institute = get_institutes(variable) - if institute: - variable['institute'] = institute - if 'activity' not in variable: - activity = get_activity(variable) - if activity: - variable['activity'] = activity - if 'sub_experiment' in variable: - subexperiment_keys = deepcopy(required_keys) - subexperiment_keys.update({'sub_experiment'}) - check.variable(variable, subexperiment_keys) - else: - check.variable(variable, required_keys) - if variable['project'] == 'obs4mips': - logger.warning("Correcting capitalization, project 'obs4mips'" - " should be written as 'obs4MIPs'") - variable['project'] = 'obs4MIPs' - variables = self._expand_tag(variables, 'ensemble') - variables = self._expand_tag(variables, 'sub_experiment') - - return variables - - def _initialize_preprocessor_output(self, diagnostic_name, raw_variables, - raw_datasets): - """Define variables in diagnostic.""" - logger.debug("Populating list of variables for diagnostic %s", - diagnostic_name) - - preprocessor_output = {} - - for variable_group, raw_variable in raw_variables.items(): - if raw_variable is None: - raw_variable = {} - else: - raw_variable = deepcopy(raw_variable) - raw_variable['variable_group'] = variable_group - if 'short_name' not in raw_variable: - raw_variable['short_name'] = variable_group - raw_variable['diagnostic'] = diagnostic_name - raw_variable['preprocessor'] = str( - raw_variable.get('preprocessor', 'default')) - preprocessor_output[variable_group] = \ - self._initialize_variables(raw_variable, raw_datasets) - - self._set_alias(preprocessor_output) - - return preprocessor_output - - def _set_alias(self, preprocessor_output): - """Add unique alias for datasets. - - Generates a unique alias for each dataset that will be shared by all - variables. Tries to make it as small as possible to make it useful for - plot legends, filenames and such - - It is composed using the keys in Recipe.info_keys that differ from - dataset to dataset. Once a diverging key is found, others are added - to the alias only if the previous ones where not enough to fully - identify the dataset. - - If key values are not strings, they will be joint using '-' if they - are iterables or replaced by they string representation if they are not - - Function will not modify alias if it is manually added to the recipe - but it will use the dataset info to compute the others - - Examples - -------- - - {project: CMIP5, model: EC-Earth, ensemble: r1i1p1} - - {project: CMIP6, model: EC-Earth, ensemble: r1i1p1f1} - will generate alias 'CMIP5' and 'CMIP6' - - - {project: CMIP5, model: EC-Earth, experiment: historical} - - {project: CMIP5, model: MPI-ESM, experiment: piControl} - will generate alias 'EC-Earth,' and 'MPI-ESM' - - - {project: CMIP5, model: EC-Earth, experiment: historical} - - {project: CMIP5, model: EC-Earth, experiment: piControl} - will generate alias 'historical' and 'piControl' - - - {project: CMIP5, model: EC-Earth, experiment: historical} - - {project: CMIP6, model: EC-Earth, experiment: historical} - - {project: CMIP5, model: MPI-ESM, experiment: historical} - - {project: CMIP6, model: MPI-ESM experiment: historical} - will generate alias 'CMIP5_EC-EARTH', 'CMIP6_EC-EARTH', 'CMIP5_MPI-ESM' - and 'CMIP6_MPI-ESM' - - - {project: CMIP5, model: EC-Earth, experiment: historical} - will generate alias 'EC-Earth' - - Parameters - ---------- - preprocessor_output : dict - preprocessor output dictionary - """ - datasets_info = set() - - def _key_str(obj): - if isinstance(obj, str): - return obj - try: - return '-'.join(obj) - except TypeError: - return str(obj) - - for variable in preprocessor_output.values(): - for dataset in variable: - alias = tuple( - _key_str(dataset.get(key, None)) for key in self.info_keys) - datasets_info.add(alias) - if 'alias' not in dataset: - dataset['alias'] = alias - - alias = dict() - for info in datasets_info: - alias[info] = [] - - datasets_info = list(datasets_info) - self._get_next_alias(alias, datasets_info, 0) - - for info in datasets_info: - alias[info] = '_'.join( - [str(value) for value in alias[info] if value is not None]) - if not alias[info]: - alias[info] = info[self.info_keys.index('dataset')] - - for variable in preprocessor_output.values(): - for dataset in variable: - dataset['alias'] = alias.get(dataset['alias'], - dataset['alias']) - - @classmethod - def _get_next_alias(cls, alias, datasets_info, i): - if i >= len(cls.info_keys): - return - key_values = set(info[i] for info in datasets_info) - if len(key_values) == 1: - for info in iter(datasets_info): - alias[info].append(None) - else: - for info in datasets_info: - alias[info].append(info[i]) - for key in key_values: - cls._get_next_alias( - alias, - [info for info in datasets_info if info[i] == key], - i + 1, - ) - def _initialize_scripts(self, diagnostic_name, raw_scripts, variable_names): """Define script in diagnostic.""" @@ -1602,18 +1064,20 @@ def _initialize_scripts(self, diagnostic_name, raw_scripts, settings['script'] = script_name # Add output dirs to settings for dir_name in ('run_dir', 'plot_dir', 'work_dir'): - settings[dir_name] = os.path.join(self._cfg[dir_name], - diagnostic_name, script_name) + settings[dir_name] = os.path.join( + getattr(self.session, dir_name), diagnostic_name, + script_name) # Copy other settings - if self._cfg['write_ncl_interface']: - settings['exit_on_ncl_warning'] = self._cfg['exit_on_warning'] + if self.session['write_ncl_interface']: + settings['exit_on_ncl_warning'] = self.session[ + 'exit_on_warning'] for key in ( 'output_file_type', 'log_level', 'profile_diagnostic', 'auxiliary_data_dir', ): - settings[key] = self._cfg[key] + settings[key] = self.session[key] scripts[script_name] = { 'script': script, @@ -1639,8 +1103,8 @@ def _resolve_diagnostic_ancestors(self, tasks): ancestor_ids = fnmatch.filter(tasks, id_glob) if not ancestor_ids: raise RecipeError( - "Could not find any ancestors matching {}". - format(id_glob)) + "Could not find any ancestors matching " + f"'{id_glob}'.") logger.debug("Pattern %s matches %s", id_glob, ancestor_ids) ancestors.extend(tasks[a] for a in ancestor_ids) @@ -1648,7 +1112,7 @@ def _resolve_diagnostic_ancestors(self, tasks): def _get_tasks_to_run(self): """Get tasks filtered and add ancestors if needed.""" - tasknames_to_run = self._cfg.get('diagnostics', []) + tasknames_to_run = self.session['diagnostics'] if tasknames_to_run: tasknames_to_run = set(tasknames_to_run) while self._update_with_ancestors(tasknames_to_run): @@ -1690,7 +1154,7 @@ def _create_diagnostic_tasks(self, diagnostic_name, diagnostic, """Create diagnostic tasks.""" tasks = [] - if self._cfg.get('run_diagnostic', True): + if self.session['run_diagnostic']: for script_name, script_cfg in diagnostic['scripts'].items(): task_name = diagnostic_name + TASKSEP + script_name @@ -1715,57 +1179,14 @@ def _create_diagnostic_tasks(self, diagnostic_name, diagnostic, return tasks - def _fill_wildcards(self, variable_group, preprocessor_output): - """Fill wildcards in the `timerange` . - - The new values will be datetime values that have been found for - the first and/or last available points. - """ - # To be generalised for other tags - datasets = self._raw_recipe.get('datasets') - diagnostics = self._raw_recipe.get('diagnostics') - additional_datasets = [] - if diagnostics: - additional_datasets = nested_lookup('additional_datasets', - diagnostics) - - raw_dataset_tags = nested_lookup('timerange', datasets) - raw_diagnostic_tags = nested_lookup('timerange', diagnostics) - - wildcard = False - for raw_timerange in raw_dataset_tags + raw_diagnostic_tags: - if '*' in raw_timerange: - wildcard = True - break - - if wildcard: - if not self._updated_recipe: - self._updated_recipe = deepcopy(self._raw_recipe) - nested_delete(self._updated_recipe, 'datasets', in_place=True) - nested_delete(self._updated_recipe, - 'additional_datasets', - in_place=True) - updated_datasets = [] - dataset_keys = set( - get_all_keys(datasets) + get_all_keys(additional_datasets) + - ['timerange']) - for data in preprocessor_output[variable_group]: - diagnostic = data['diagnostic'] - updated_datasets.append( - {key: data[key] - for key in dataset_keys if key in data}) - self._updated_recipe['diagnostics'][diagnostic]['variables'][ - variable_group].pop('timerange', None) - self._updated_recipe['diagnostics'][diagnostic]['variables'][ - variable_group].update( - {'additional_datasets': updated_datasets}) - def _create_preprocessor_tasks(self, diagnostic_name, diagnostic, tasknames_to_run, any_diag_script_is_run): """Create preprocessor tasks.""" tasks = [] failed_tasks = [] - for variable_group in diagnostic['preprocessor_output']: + for variable_group, datasets in groupby( + diagnostic['datasets'], + key=lambda ds: ds.facets['variable_group']): task_name = diagnostic_name + TASKSEP + variable_group # Skip preprocessor if not a single diagnostic script is run and @@ -1781,7 +1202,7 @@ def _create_preprocessor_tasks(self, diagnostic_name, diagnostic, continue # Resume previous runs if requested, else create a new task - for resume_dir in self._cfg['resume_from']: + for resume_dir in self.session['resume_from']: prev_preproc_dir = Path( resume_dir, 'preproc', @@ -1792,8 +1213,7 @@ def _create_preprocessor_tasks(self, diagnostic_name, diagnostic, logger.info("Re-using preprocessed files from %s for %s", prev_preproc_dir, task_name) preproc_dir = Path( - self._cfg['preproc_dir'], - 'preproc', + self.session.preproc_dir, diagnostic_name, variable_group, ) @@ -1804,17 +1224,13 @@ def _create_preprocessor_tasks(self, diagnostic_name, diagnostic, logger.info("Creating preprocessor task %s", task_name) try: task = _get_preprocessor_task( - variables=diagnostic['preprocessor_output'] - [variable_group], + datasets=list(datasets), profiles=self._preprocessors, - config_user=self._cfg, task_name=task_name, ) - except RecipeError as ex: - failed_tasks.append(ex) + except RecipeError as exc: + failed_tasks.append(exc) else: - self._fill_wildcards(variable_group, - diagnostic['preprocessor_output']) tasks.append(task) return tasks, failed_tasks @@ -1861,7 +1277,7 @@ def _create_tasks(self): check.tasks_valid(tasks) # Resolve diagnostic ancestors - if self._cfg.get('run_diagnostic', True): + if self.session['run_diagnostic']: self._resolve_diagnostic_ancestors(tasks) return tasks @@ -1889,15 +1305,18 @@ def __str__(self): def run(self): """Run all tasks in the recipe.""" - self.write_filled_recipe() if not self.tasks: raise RecipeError('No tasks to run!') + filled_recipe = self.write_filled_recipe() # Download required data - if not self._cfg['offline']: - esgf.download(self._download_files, self._cfg['download_dir']) + if not self.session['offline']: + esgf.download(self._download_files, self.session['download_dir']) - self.tasks.run(max_parallel_tasks=self._cfg['max_parallel_tasks']) + self.tasks.run(max_parallel_tasks=self.session['max_parallel_tasks']) + logger.info( + "Wrote recipe with version numbers and wildcards " + "to:\nfile://%s", filled_recipe) self.write_html_summary() def get_output(self) -> dict: @@ -1910,13 +1329,13 @@ def get_output(self) -> dict: """ output = {} - output['recipe_config'] = self._cfg + output['session'] = self.session output['recipe_filename'] = self._filename output['recipe_data'] = self._raw_recipe output['task_output'] = {} for task in sorted(self.tasks.flatten(), key=lambda t: t.priority): - if self._cfg['remove_preproc_dir'] and isinstance( + if self.session['remove_preproc_dir'] and isinstance( task, PreprocessingTask): # Skip preprocessing tasks that are deleted afterwards continue @@ -1926,13 +1345,14 @@ def get_output(self) -> dict: def write_filled_recipe(self): """Write copy of recipe with filled wildcards.""" - if self._updated_recipe: - run_dir = self._cfg['run_dir'] - filename = self._filename.split('.') - filename[0] = filename[0] + '_filled' - new_filename = '.'.join(filename) - with open(os.path.join(run_dir, new_filename), 'w') as file: - yaml.safe_dump(self._updated_recipe, file, sort_keys=False) + recipe = datasets_to_recipe(USED_DATASETS, self._raw_recipe) + filename = self.session.run_dir / f"{self._filename.stem}_filled.yml" + with filename.open('w', encoding='utf-8') as file: + yaml.safe_dump(recipe, file, sort_keys=False) + logger.info( + "Wrote recipe with version numbers and wildcards " + "to:\nfile://%s", filename) + return filename def write_html_summary(self): """Write summary html file to the output dir.""" diff --git a/esmvalcore/_recipe/to_datasets.py b/esmvalcore/_recipe/to_datasets.py new file mode 100644 index 0000000000..06423cbed6 --- /dev/null +++ b/esmvalcore/_recipe/to_datasets.py @@ -0,0 +1,515 @@ +"""Module that contains functions for reading the `Dataset`s from a recipe.""" +from __future__ import annotations + +import logging +from copy import deepcopy +from numbers import Number +from pathlib import Path +from typing import Any, Iterable, Iterator + +from esmvalcore.cmor.table import _CMOR_KEYS, _update_cmor_facets +from esmvalcore.config import Session +from esmvalcore.dataset import Dataset, _isglob +from esmvalcore.esgf.facets import FACETS +from esmvalcore.exceptions import RecipeError +from esmvalcore.local import LocalFile, _replace_years_with_timerange +from esmvalcore.preprocessor._derive import get_required +from esmvalcore.preprocessor._io import DATASET_KEYS +from esmvalcore.preprocessor._supplementary_vars import ( + PREPROCESSOR_SUPPLEMENTARIES, +) +from esmvalcore.typing import Facets, FacetValue + +from . import check +from ._io import _load_recipe + +logger = logging.getLogger(__name__) + +_ALIAS_INFO_KEYS = ( + 'project', + 'activity', + 'driver', + 'dataset', + 'exp', + 'sub_experiment', + 'ensemble', + 'version', +) +"""List of keys to be used to compose the alias, ordered by priority.""" + + +def _facet_to_str(facet_value: FacetValue) -> str: + """Get a string representation of a facet value.""" + if isinstance(facet_value, str): + return facet_value + if isinstance(facet_value, Iterable): + return '-'.join(str(v) for v in facet_value) + return str(facet_value) + + +def _set_alias(variables): + """Add unique alias for datasets. + + Generates a unique alias for each dataset that will be shared by all + variables. Tries to make it as small as possible to make it useful for + plot legends, filenames and such + + It is composed using the keys in Recipe.info_keys that differ from + dataset to dataset. Once a diverging key is found, others are added + to the alias only if the previous ones where not enough to fully + identify the dataset. + + If key values are not strings, they will be joint using '-' if they + are iterables or replaced by they string representation if they are not + + Function will not modify alias if it is manually added to the recipe + but it will use the dataset info to compute the others + + Examples + -------- + - {project: CMIP5, model: EC-Earth, ensemble: r1i1p1} + - {project: CMIP6, model: EC-Earth, ensemble: r1i1p1f1} + will generate alias 'CMIP5' and 'CMIP6' + + - {project: CMIP5, model: EC-Earth, experiment: historical} + - {project: CMIP5, model: MPI-ESM, experiment: piControl} + will generate alias 'EC-Earth,' and 'MPI-ESM' + + - {project: CMIP5, model: EC-Earth, experiment: historical} + - {project: CMIP5, model: EC-Earth, experiment: piControl} + will generate alias 'historical' and 'piControl' + + - {project: CMIP5, model: EC-Earth, experiment: historical} + - {project: CMIP6, model: EC-Earth, experiment: historical} + - {project: CMIP5, model: MPI-ESM, experiment: historical} + - {project: CMIP6, model: MPI-ESM experiment: historical} + will generate alias 'CMIP5_EC-EARTH', 'CMIP6_EC-EARTH', 'CMIP5_MPI-ESM' + and 'CMIP6_MPI-ESM' + + - {project: CMIP5, model: EC-Earth, experiment: historical} + will generate alias 'EC-Earth' + + Parameters + ---------- + variables : list + for each recipe variable, a list of datasets + """ + datasets_info = set() + + for variable in variables: + for dataset in variable: + alias = tuple( + _facet_to_str(dataset.facets.get(key, None)) + for key in _ALIAS_INFO_KEYS) + datasets_info.add(alias) + if 'alias' not in dataset.facets: + dataset.facets['alias'] = alias + + alias = {} + for info in datasets_info: + alias[info] = [] + + datasets_info = list(datasets_info) + _get_next_alias(alias, datasets_info, 0) + + for info in datasets_info: + alias[info] = '_'.join( + [str(value) for value in alias[info] if value is not None]) + if not alias[info]: + alias[info] = info[_ALIAS_INFO_KEYS.index('dataset')] + + for variable in variables: + for dataset in variable: + dataset.facets['alias'] = alias.get(dataset.facets['alias'], + dataset.facets['alias']) + + +def _get_next_alias(alias, datasets_info, i): + if i >= len(_ALIAS_INFO_KEYS): + return + key_values = set(info[i] for info in datasets_info) + if len(key_values) == 1: + for info in iter(datasets_info): + alias[info].append(None) + else: + for info in datasets_info: + alias[info].append(info[i]) + for key in key_values: + _get_next_alias( + alias, + [info for info in datasets_info if info[i] == key], + i + 1, + ) + + +def _check_supplementaries_valid(supplementaries: Iterable[Facets]) -> None: + """Check that supplementary variables have a short_name.""" + for facets in supplementaries: + if 'short_name' not in facets: + raise RecipeError( + "'short_name' is required for supplementary_variables " + f"entries, but missing in {facets}") + + +def _merge_supplementary_dicts( + var_facets: Iterable[Facets], + ds_facets: Iterable[Facets], +) -> list[Facets]: + """Merge the elements of `var_facets` with those in `ds_facets`. + + Both are lists of dicts containing facets + """ + _check_supplementaries_valid(var_facets) + _check_supplementaries_valid(ds_facets) + merged = {} + for facets in var_facets: + merged[facets['short_name']] = facets + for facets in ds_facets: + short_name = facets['short_name'] + if short_name not in merged: + merged[short_name] = {} + merged[short_name].update(facets) + return list(merged.values()) + + +def _fix_cmip5_fx_ensemble(dataset: Dataset): + """Automatically correct the wrong ensemble for CMIP5 fx variables.""" + if (dataset.facets.get('project') == 'CMIP5' + and dataset.facets.get('mip') == 'fx' + and dataset.facets.get('ensemble') != 'r0i0p0' + and not dataset.files): + original_ensemble = dataset['ensemble'] + copy = dataset.copy() + copy.facets['ensemble'] = 'r0i0p0' + if copy.files: + dataset.facets['ensemble'] = 'r0i0p0' + logger.info("Corrected wrong 'ensemble' from '%s' to '%s' for %s", + original_ensemble, dataset['ensemble'], + dataset.summary(shorten=True)) + dataset.find_files() + + +def _get_supplementary_short_names( + facets: Facets, + step: str, +) -> list[str]: + """Get the most applicable supplementary short_names.""" + # Determine if the main variable is an ocean variable. + var_facets = dict(facets) + _update_cmor_facets(var_facets) + realms = var_facets.get('modeling_realm', []) + if isinstance(realms, (str, Number)): + realms = [str(realms)] + ocean_realms = {'ocean', 'seaIce', 'ocnBgchem'} + is_ocean_variable = any(realm in ocean_realms for realm in realms) + + # Guess the best matching supplementary variable based on the realm. + short_names = PREPROCESSOR_SUPPLEMENTARIES[step]['variables'] + if set(short_names) == {'areacella', 'areacello'}: + short_names = ['areacello'] if is_ocean_variable else ['areacella'] + if set(short_names) == {'sftlf', 'sftof'}: + short_names = ['sftof'] if is_ocean_variable else ['sftlf'] + + return short_names + + +def _append_missing_supplementaries( + supplementaries: list[Facets], + facets: Facets, + settings: dict[str, Any], +) -> None: + """Append wildcard definitions for missing supplementary variables.""" + steps = [step for step in settings if step in PREPROCESSOR_SUPPLEMENTARIES] + + project: str = facets['project'] # type: ignore + for step in steps: + for short_name in _get_supplementary_short_names(facets, step): + short_names = {f['short_name'] for f in supplementaries} + if short_name in short_names: + continue + + supplementary_facets: Facets = { + facet: '*' + for facet in FACETS.get(project, ['mip']) + if facet not in _CMOR_KEYS + } + if 'version' in facets: + supplementary_facets['version'] = '*' + supplementary_facets['short_name'] = short_name + supplementaries.append(supplementary_facets) + + +def _get_dataset_facets_from_recipe( + variable_group: str, + recipe_variable: dict[str, Any], + recipe_dataset: dict[str, Any], + profiles: dict[str, Any], + session: Session, +) -> tuple[Facets, list[Facets]]: + """Read the facets for a single dataset definition from the recipe.""" + facets = deepcopy(recipe_variable) + facets.pop('additional_datasets', None) + recipe_dataset = deepcopy(recipe_dataset) + + supplementaries = _merge_supplementary_dicts( + facets.pop('supplementary_variables', []), + recipe_dataset.pop('supplementary_variables', []), + ) + + facets.update(recipe_dataset) + + if 'short_name' not in facets: + facets['short_name'] = variable_group + + # Flaky support for limiting the number of years in a recipe. + # If we want this to work, it should actually be done based on `timerange`, + # after any wildcards have been resolved. + if 'end_year' in facets and session['max_years']: + facets['end_year'] = min( + facets['end_year'], + facets['start_year'] + session['max_years'] - 1) + + # Legacy: support start_year and end_year instead of timerange + _replace_years_with_timerange(facets) + + # Legacy: support wrong capitalization of obs4MIPs + if facets['project'] == 'obs4mips': + logger.warning("Correcting capitalization, project 'obs4mips' " + "should be written as 'obs4MIPs'") + facets['project'] = 'obs4MIPs' + + check.variable( + facets, + required_keys=( + 'short_name', + 'mip', + 'dataset', + 'project', + ), + ) + + if not session['use_legacy_supplementaries']: + preprocessor = facets.get('preprocessor', 'default') + settings = profiles.get(preprocessor, {}) + _append_missing_supplementaries(supplementaries, facets, settings) + supplementaries = [ + facets for facets in supplementaries + if not facets.pop('skip', False) + ] + + return facets, supplementaries + + +def _get_facets_from_recipe( + recipe: dict[str, Any], + diagnostic_name: str, + variable_group: str, + session: Session, +) -> Iterator[tuple[Facets, list[Facets]]]: + """Read the facets for the detasets of one variable from the recipe.""" + diagnostic = recipe['diagnostics'][diagnostic_name] + recipe_variable = diagnostic['variables'][variable_group] + if recipe_variable is None: + recipe_variable = {} + + recipe_datasets = (recipe.get('datasets', []) + + diagnostic.get('additional_datasets', []) + + recipe_variable.get('additional_datasets', [])) + check.duplicate_datasets(recipe_datasets, diagnostic_name, variable_group) + + # The NCL interface requires a distinction between variable and + # dataset keys as defined in the recipe. `DATASET_KEYS` is used to + # keep track of which keys are part of the dataset. + DATASET_KEYS.update(key for ds in recipe_datasets for key in ds) + + profiles = recipe.setdefault('preprocessors', {'default': {}}) + + for recipe_dataset in recipe_datasets: + yield _get_dataset_facets_from_recipe( + variable_group=variable_group, + recipe_variable=recipe_variable, + recipe_dataset=recipe_dataset, + profiles=profiles, + session=session, + ) + + +def _get_datasets_for_variable( + recipe: dict[str, Any], + diagnostic_name: str, + variable_group: str, + session: Session, +) -> list[Dataset]: + """Read the datasets from a variable definition in the recipe.""" + logger.debug( + "Populating list of datasets for variable %s in " + "diagnostic %s", variable_group, diagnostic_name) + + datasets = [] + idx = 0 + + for facets, supplementaries in _get_facets_from_recipe( + recipe, + diagnostic_name=diagnostic_name, + variable_group=variable_group, + session=session, + ): + template0 = Dataset(**facets) + template0.session = session + for template1 in template0.from_ranges(): + for supplementary_facets in supplementaries: + template1.add_supplementary(**supplementary_facets) + for supplementary_ds in template1.supplementaries: + supplementary_ds.facets.pop('preprocessor', None) + for dataset in _dataset_from_files(template1): + dataset['variable_group'] = variable_group + dataset['diagnostic'] = diagnostic_name + dataset['recipe_dataset_index'] = idx # type: ignore + logger.debug("Found %s", dataset.summary(shorten=True)) + datasets.append(dataset) + idx += 1 + + return datasets + + +def datasets_from_recipe( + recipe: Path | str | dict[str, Any], + session: Session, +) -> list[Dataset]: + """Read datasets from a recipe.""" + datasets = [] + + recipe = _load_recipe(recipe) + diagnostics = recipe.get('diagnostics') or {} + for name, diagnostic in diagnostics.items(): + diagnostic_datasets = [] + for variable_group in diagnostic.get('variables', {}): + variable_datasets = _get_datasets_for_variable( + recipe, + diagnostic_name=name, + variable_group=variable_group, + session=session, + ) + diagnostic_datasets.append(variable_datasets) + datasets.extend(variable_datasets) + + _set_alias(diagnostic_datasets) + + return datasets + + +def _dataset_from_files(dataset: Dataset) -> list[Dataset]: + """Replace facet values of '*' based on available files.""" + result: list[Dataset] = [] + errors = [] + + if any(_isglob(f) for f in dataset.facets.values()): + logger.debug( + "Expanding dataset globs for dataset %s, " + "this may take a while..", dataset.summary(shorten=True)) + + repr_dataset = _representative_dataset(dataset) + for repr_ds in repr_dataset.from_files(): + updated_facets = {} + failed = {} + for key, value in dataset.facets.items(): + if _isglob(value): + if key in repr_ds.facets and not _isglob(repr_ds[key]): + updated_facets[key] = repr_ds.facets[key] + else: + failed[key] = value + + if failed: + msg = ("Unable to replace " + + ", ".join(f"{k}={v}" for k, v in failed.items()) + + f" by a value for\n{dataset}") + # Set supplementaries to [] to avoid searching for supplementary + # files. + repr_ds.supplementaries = [] + if repr_ds.files: + paths_msg = "paths to " if any( + isinstance(f, LocalFile) for f in repr_ds.files) else "" + msg = (f"{msg}\nDo the {paths_msg}the files:\n" + + "\n".join(f"{f} with facets: {f.facets}" + for f in repr_ds.files) + + "\nprovide the missing facet values?") + else: + timerange = repr_ds.facets.get('timerange') + patterns = repr_ds._file_globs + msg = ( + f"{msg}\nNo files found matching:\n" + + "\n".join(str(p) for p in patterns) + # type:ignore + (f"\nwithin the requested timerange {timerange}." + if timerange else "")) + errors.append(msg) + continue + + new_ds = dataset.copy() + new_ds.facets.update(updated_facets) + new_ds.supplementaries = repr_ds.supplementaries + result.append(new_ds) + + if errors: + raise RecipeError("\n".join(errors)) + + return result + + +def _derive_needed(dataset: Dataset) -> bool: + """Check if dataset needs to be derived from other datasets.""" + if not dataset.facets.get('derive'): + return False + if dataset.facets.get('force_derivation'): + return True + if _isglob(dataset.facets.get('timerange', '')): + # Our file finding routines are not able to handle globs. + dataset = dataset.copy() + dataset.facets.pop('timerange') + + copy = dataset.copy() + copy.supplementaries = [] + return not copy.files + + +def _get_input_datasets(dataset: Dataset) -> list[Dataset]: + """Determine the input datasets needed for deriving `dataset`.""" + facets = dataset.facets + if not _derive_needed(dataset): + _fix_cmip5_fx_ensemble(dataset) + return [dataset] + + # Configure input datasets needed to derive variable + datasets = [] + required_vars = get_required(facets['short_name'], facets['project']) + # idea: add option to specify facets in list of dicts that is value of + # 'derive' in the recipe and use that instead of get_required? + for input_facets in required_vars: + input_dataset = dataset.copy(**input_facets) + _update_cmor_facets(input_dataset.facets, override=True) + input_dataset.augment_facets() + _fix_cmip5_fx_ensemble(input_dataset) + if input_facets.get('optional') and not input_dataset.files: + logger.info( + "Skipping: no data found for %s which is marked as " + "'optional'", input_dataset) + else: + datasets.append(input_dataset) + + # Check timeranges of available input data. + timeranges = set() + for input_dataset in datasets: + if 'timerange' in input_dataset.facets: + timeranges.add(input_dataset.facets['timerange']) + check.differing_timeranges(timeranges, required_vars) + + return datasets + + +def _representative_dataset(dataset: Dataset) -> Dataset: + """Find a representative dataset that has files available.""" + copy = dataset.copy() + copy.supplementaries = [] + datasets = _get_input_datasets(copy) + representative_dataset = datasets[0] + representative_dataset.supplementaries = dataset.supplementaries + return representative_dataset diff --git a/esmvalcore/cmor/_fixes/fix.py b/esmvalcore/cmor/_fixes/fix.py index 20840d45c1..db41dc3ff8 100644 --- a/esmvalcore/cmor/_fixes/fix.py +++ b/esmvalcore/cmor/_fixes/fix.py @@ -2,6 +2,7 @@ import importlib import inspect import os +from pathlib import Path from ..table import CMOR_TABLES @@ -25,7 +26,7 @@ def __init__(self, vardef, extra_facets=None): extra_facets = {} self.extra_facets = extra_facets - def fix_file(self, filepath, output_dir): + def fix_file(self, filepath: Path, output_dir: Path) -> Path: """Apply fixes to the files prior to creating the cube. Should be used only to fix errors that prevent loading or can @@ -34,14 +35,14 @@ def fix_file(self, filepath, output_dir): Parameters ---------- - filepath: str + filepath: Path file to fix - output_dir: str + output_dir: Path path to the folder to store the fixed files, if required Returns ------- - str + Path Path to the corrected file. It can be different from the original filepath if a fix has been applied, but if not it should be the original filepath diff --git a/esmvalcore/cmor/fix.py b/esmvalcore/cmor/fix.py index 5775a1f3f9..ba10e28680 100644 --- a/esmvalcore/cmor/fix.py +++ b/esmvalcore/cmor/fix.py @@ -6,6 +6,7 @@ """ import logging from collections import defaultdict +from pathlib import Path from iris.cube import CubeList @@ -15,8 +16,15 @@ logger = logging.getLogger(__name__) -def fix_file(file, short_name, project, dataset, mip, output_dir, - **extra_facets): +def fix_file( + file: Path, + short_name: str, + project: str, + dataset: str, + mip: str, + output_dir: Path, + **extra_facets, +) -> Path: """Fix files before ESMValTool can load them. This fixes are only for issues that prevent iris from loading the cube or @@ -26,7 +34,7 @@ def fix_file(file, short_name, project, dataset, mip, output_dir, Parameters ---------- - file: str + file: Path Path to the original file. short_name: str Variable's short name. @@ -36,7 +44,7 @@ def fix_file(file, short_name, project, dataset, mip, output_dir, Name of the dataset. mip: str Variable's MIP. - output_dir: str + output_dir: Path Output directory for fixed files. **extra_facets: dict, optional Extra facets are mainly used for data outside of the big projects like @@ -44,9 +52,11 @@ def fix_file(file, short_name, project, dataset, mip, output_dir, Returns ------- - str: + Path: Path to the fixed file. """ + if not output_dir.exists(): + output_dir.mkdir(parents=True, exist_ok=True) # Update extra_facets with variable information given as regular arguments # to this function extra_facets.update({ diff --git a/esmvalcore/cmor/table.py b/esmvalcore/cmor/table.py index 10b55a1dc5..da0333c412 100644 --- a/esmvalcore/cmor/table.py +++ b/esmvalcore/cmor/table.py @@ -65,6 +65,13 @@ def _update_cmor_facets(facets, override=False): facets) +def _get_mips(project: str, short_name: str) -> list[str]: + """Get all available MIP tables in a project.""" + tables = CMOR_TABLES[project].tables + mips = [mip for mip in tables if short_name in tables[mip]] + return mips + + def get_var_info(project, mip, short_name): """Get variable information. diff --git a/esmvalcore/config/_config.py b/esmvalcore/config/_config.py index 595a4ded00..4cd0b57aa6 100644 --- a/esmvalcore/config/_config.py +++ b/esmvalcore/config/_config.py @@ -53,9 +53,12 @@ def _load_extra_facets(project, extra_facets_dir): return config -def get_extra_facets(project, dataset, mip, short_name, extra_facets_dir): +def get_extra_facets(dataset, extra_facets_dir): """Read configuration files with additional variable information.""" - project_details = _load_extra_facets(project, extra_facets_dir) + project_details = _load_extra_facets( + dataset.facets['project'], + extra_facets_dir, + ) def pattern_filter(patterns, name): """Get the subset of the list `patterns` that `name` matches. @@ -75,10 +78,10 @@ def pattern_filter(patterns, name): return [pat for pat in patterns if fnmatch.fnmatchcase(name, pat)] extra_facets = {} - for dataset_ in pattern_filter(project_details, dataset): - for mip_ in pattern_filter(project_details[dataset_], mip): + for dataset_ in pattern_filter(project_details, dataset['dataset']): + for mip_ in pattern_filter(project_details[dataset_], dataset['mip']): for var in pattern_filter(project_details[dataset_][mip_], - short_name): + dataset['short_name']): facets = project_details[dataset_][mip_][var] extra_facets.update(facets) diff --git a/esmvalcore/config/_config_object.py b/esmvalcore/config/_config_object.py index 52ff2d5019..1d2c94e1e9 100644 --- a/esmvalcore/config/_config_object.py +++ b/esmvalcore/config/_config_object.py @@ -75,23 +75,24 @@ def _load_default_config(cls, filename: Union[os.PathLike, str]): mapping = _read_config_file(filename) # Add defaults that are not available in esmvalcore/config-user.yml + mapping['check_level'] = CheckLevels.DEFAULT mapping['config_file'] = filename mapping['diagnostics'] = None mapping['extra_facets_dir'] = tuple() - mapping['resume_from'] = [] - mapping['check_level'] = CheckLevels.DEFAULT mapping['max_datasets'] = None mapping['max_years'] = None + mapping['resume_from'] = [] mapping['run_diagnostic'] = True mapping['skip_nonexistent'] = False + mapping['use_legacy_supplementaries'] = None new.update(mapping) return new def load_from_file( - self, - filename: Optional[Union[os.PathLike, str]] = None, + self, + filename: Optional[Union[os.PathLike, str]] = None, ) -> None: """Load user configuration from the given file.""" if filename is None: diff --git a/esmvalcore/config/_config_validators.py b/esmvalcore/config/_config_validators.py index f202c80a9d..a77c367084 100644 --- a/esmvalcore/config/_config_validators.py +++ b/esmvalcore/config/_config_validators.py @@ -5,9 +5,9 @@ import os.path import warnings from collections.abc import Iterable -from functools import lru_cache +from functools import lru_cache, partial from pathlib import Path -from typing import Optional, Union +from typing import Any, Optional, Union from esmvalcore import __version__ as current_version from esmvalcore.cmor.check import CheckLevels @@ -16,7 +16,10 @@ importlib_files, load_config_developer, ) -from esmvalcore.exceptions import ESMValCoreDeprecationWarning +from esmvalcore.exceptions import ( + ESMValCoreDeprecationWarning, + InvalidConfigParameter, +) logger = logging.getLogger(__name__) @@ -35,6 +38,7 @@ def _make_type_validator(cls, *, allow_none=False): Return a validator that converts inputs to *cls* or raises (and possibly allows ``None`` as well). """ + def validator(inp): looks_like_none = isinstance(inp, str) and (inp.lower() == "none") if (allow_none and (inp is None or looks_like_none)): @@ -68,6 +72,7 @@ def _listify_validator(scalar_validator, docstring=None, return_type=list): """Apply the validator to a list.""" + def func(inp): if isinstance(inp, str): try: @@ -144,6 +149,7 @@ def validate_positive(value): def _chain_validator(*funcs): """Chain a series of validators.""" + def chained(value): for func in funcs: value = func(value) @@ -156,6 +162,8 @@ def chained(value): validate_string_or_none = _make_type_validator(str, allow_none=True) validate_stringlist = _listify_validator(validate_string, docstring='Return a list of strings.') + +validate_bool_or_none = partial(validate_bool, allow_none=True) validate_int = _make_type_validator(int) validate_int_or_none = _make_type_validator(int, allow_none=True) validate_float = _make_type_validator(float) @@ -232,7 +240,7 @@ def validate_check_level(value): def validate_diagnostics( - diagnostics: Union[Iterable[str], str, None], + diagnostics: Union[Iterable[str], str, None] ) -> Optional[set[str]]: """Validate diagnostic location.""" if diagnostics is None: @@ -245,69 +253,84 @@ def validate_diagnostics( } -def deprecate(func, variable, version: Optional[str] = None): - """Wrap function to mark variables to be deprecated. +def deprecate(validator, option: str, default: Any, version: str): + """Wrap function to mark variables as deprecated. - This will give a warning if the function will be/has been deprecated. + This will give a warning if the function has been deprecated and raise + an exception once the parameter (should have been) removed. Parameters ---------- - func: - Validator function to wrap - variable: str - Name of the variable to deprecate - version: str - Version to deprecate the variable in, should be something - like '2.2.3' + validator: + Validator function to wrap. + option: + Name of the option to deprecate. + default: + The new default value, no warning is issued when setting this value. + version: + Version to remove the option in, should be something like '2.2.3'. """ - if not version: - version = 'a future version' - if current_version >= version: - warnings.warn(f"`{variable}` has been removed in {version}", - ESMValCoreDeprecationWarning) - else: - warnings.warn(f"`{variable}` will be removed in {version}.", - ESMValCoreDeprecationWarning, - stacklevel=2) + def get_version(version_string): + return tuple(int(i) for i in version_string.split('.')[:3]) - return func + msg_head = f"The configuration option '{option}'" + msg_tail = f"removed in {version}." + + def wrapper(value): + if get_version(current_version) >= get_version(version): + raise InvalidConfigParameter(f"{msg_head} has been {msg_tail}") + + if value != default: + warnings.warn( + f"{msg_head} will be {msg_tail}", + ESMValCoreDeprecationWarning, + ) + return validator(value) + + return wrapper +_validate_use_legacy_supplementaries = deprecate( + validator=validate_bool_or_none, + option='use_legacy_supplementaries', + default=None, + version='2.10.0', +) + _validators = { # From user config - 'log_level': validate_string, - 'exit_on_warning': validate_bool, - 'output_dir': validate_path, - 'download_dir': validate_path, + 'always_search_esgf': validate_bool, 'auxiliary_data_dir': validate_path, - 'extra_facets_dir': validate_pathtuple, 'compress_netcdf': validate_bool, - 'save_intermediary_cubes': validate_bool, - 'remove_preproc_dir': validate_bool, - 'max_parallel_tasks': validate_int_or_none, 'config_developer_file': validate_config_developer, + 'download_dir': validate_path, + 'drs': validate_drs, + 'exit_on_warning': validate_bool, + 'extra_facets_dir': validate_pathtuple, + 'log_level': validate_string, + 'max_parallel_tasks': validate_int_or_none, + 'offline': validate_bool, + 'output_dir': validate_path, + 'output_file_type': validate_string, 'profile_diagnostic': validate_bool, + 'remove_preproc_dir': validate_bool, + 'rootpath': validate_rootpath, 'run_diagnostic': validate_bool, - 'output_file_type': validate_string, - "offline": validate_bool, - 'always_search_esgf': validate_bool, + 'save_intermediary_cubes': validate_bool, + 'use_legacy_supplementaries': _validate_use_legacy_supplementaries, # From CLI - "resume_from": validate_pathlist, - "skip_nonexistent": validate_bool, - "diagnostics": validate_diagnostics, - "check_level": validate_check_level, + 'check_level': validate_check_level, + 'diagnostics': validate_diagnostics, 'max_years': validate_int_positive_or_none, 'max_datasets': validate_int_positive_or_none, + 'resume_from': validate_pathlist, + 'skip_nonexistent': validate_bool, # From recipe 'write_ncl_interface': validate_bool, - # oldstyle - 'rootpath': validate_rootpath, - 'drs': validate_drs, - # config location 'config_file': validate_path, } diff --git a/esmvalcore/dataset.py b/esmvalcore/dataset.py index 3926289067..ca95241d5e 100644 --- a/esmvalcore/dataset.py +++ b/esmvalcore/dataset.py @@ -8,6 +8,7 @@ import uuid from copy import deepcopy from fnmatch import fnmatchcase +from itertools import groupby from pathlib import Path from typing import Any, Iterator, Sequence, Union @@ -16,7 +17,7 @@ from esmvalcore import esgf, local from esmvalcore._recipe import check from esmvalcore._recipe.from_datasets import datasets_to_recipe -from esmvalcore.cmor.table import _update_cmor_facets +from esmvalcore.cmor.table import _get_mips, _update_cmor_facets from esmvalcore.config import CFG, Session from esmvalcore.config._config import ( get_activity, @@ -34,6 +35,7 @@ __all__ = [ 'Dataset', + 'INHERITED_FACETS', 'datasets_to_recipe', ] @@ -41,6 +43,21 @@ File = Union[esgf.ESGFFile, local.LocalFile] +INHERITED_FACETS: list[str] = [ + 'dataset', + 'domain', + 'driver', + 'grid', + 'project', + 'timerange', +] +"""Inherited facets. + +Supplementary datasets created based on the available files using the +:func:`Dataset.from_files` method will inherit the values of these facets from +the main dataset. +""" + def _augment(base: dict, update: dict): """Update dict `base` with values from dict `update`.""" @@ -73,8 +90,8 @@ class Dataset: Attributes ---------- - ancillaries : list[Dataset] - List of ancillary datasets. + supplementaries : list[Dataset] + List of supplementary datasets. facets: :obj:`esmvalcore.typing.Facets` Facets describing the dataset. """ @@ -82,7 +99,7 @@ class Dataset: def __init__(self, **facets: FacetValue): self.facets: Facets = {} - self.ancillaries: list['Dataset'] = [] + self.supplementaries: list['Dataset'] = [] self._persist: set[str] = set() self._session: Session | None = None @@ -92,40 +109,139 @@ def __init__(self, **facets: FacetValue): for key, value in facets.items(): self.set_facet(key, deepcopy(value), persist=True) - def _get_available_facets(self) -> Iterator[Facets]: - """Yield unique combinations of facets based on the available files.""" + @staticmethod + def from_recipe( + recipe: Path | str | dict, + session: Session, + ) -> list['Dataset']: + """Read datasets from a recipe. - def same(facets_a, facets_b): - """Define when two sets of facets are the same.""" - return facets_a.issubset(facets_b) or facets_b.issubset(facets_a) + Parameters + ---------- + recipe + :ref:`Recipe ` to load the datasets from. The value + provided here should be either a path to a file, a recipe file + that has been loaded using e.g. :func:`yaml.safe_load`, or an + :obj:`str` that can be loaded using :func:`yaml.safe_load`. + session + Datasets to use in the recipe. + Returns + ------- + list[Dataset] + A list of datasets. + """ + from esmvalcore._recipe.to_datasets import datasets_from_recipe + return datasets_from_recipe(recipe, session) + + def _file_to_dataset( + self, + file: esgf.ESGFFile | local.LocalFile, + ) -> Dataset: + """Create a dataset from a file with a `facets` attribute.""" + facets = dict(file.facets) + if 'version' not in self.facets: + # Remove version facet if no specific version requested + facets.pop('version', None) + + updated_facets = { + f: v + for f, v in facets.items() if f in self.facets + and _isglob(self.facets[f]) and _ismatch(v, self.facets[f]) + } dataset = self.copy() - dataset.ancillaries = [] - if _isglob(dataset.facets.get('timerange')): + dataset.facets.update(updated_facets) + + # If possible, remove unexpanded facets that can be automatically + # populated. + unexpanded = {f for f, v in dataset.facets.items() if _isglob(v)} + required_for_augment = {'project', 'mip', 'short_name', 'dataset'} + if unexpanded and not unexpanded & required_for_augment: + copy = dataset.copy() + copy.supplementaries = [] + for facet in unexpanded: + copy.facets.pop(facet) + copy.augment_facets() + for facet in unexpanded: + if facet in copy.facets: + dataset.facets.pop(facet) + + return dataset + + def _get_available_datasets(self) -> Iterator[Dataset]: + """Yield datasets based on the available files. + + This function requires that self.facets['mip'] is not a glob pattern. + """ + dataset_template = self.copy() + dataset_template.supplementaries = [] + if _isglob(dataset_template.facets.get('timerange')): # Remove wildcard `timerange` facet, because data finding cannot # handle it - dataset.facets.pop('timerange') + dataset_template.facets.pop('timerange') - seen: list[frozenset[tuple[str, FacetValue]]] = [] - for file in dataset.files: - facets = dict(file.facets) - if 'version' not in self.facets: - # Remove version facet if no specific version requested - facets.pop('version', None) - - facetset = frozenset(facets.items()) - - # Filter out identical facetsets - for prev_facetset in seen: - if same(facetset, prev_facetset): - break + seen = set() + partially_defined = [] + expanded = False + for file in dataset_template.files: + dataset = self._file_to_dataset(file) + + # Filter out identical datasets + facetset = frozenset( + (f, frozenset(v) if isinstance(v, list) else v) + for f, v in dataset.facets.items()) + if facetset not in seen: + seen.add(facetset) + if any(_isglob(v) for f, v in dataset.facets.items() + if f != 'timerange'): + partially_defined.append((dataset, file)) + else: + dataset._update_timerange() + dataset._supplementaries_from_files() + expanded = True + yield dataset + + # Only yield datasets with globs if there is no better alternative + for dataset, file in partially_defined: + msg = (f"{dataset} with unexpanded wildcards, created from file " + f"{file} with facets {file.facets}. Are the missing facets " + "in the path to the file?" if isinstance( + file, local.LocalFile) else "available on ESGF?") + if expanded: + logger.info("Ignoring %s", msg) else: - seen.append(facetset) - yield dict(facetset) + logger.debug( + "Not updating timerange and supplementaries for %s " + "because it still contains wildcards.", msg) + yield dataset def from_files(self) -> Iterator['Dataset']: """Create datasets based on the available files. + The facet values for local files are retrieved from the directory tree + where the directories represent the facets values. + Reading facet values from file names is not yet supported. + See :ref:`CMOR-DRS` for more information on this kind of file + organization. + + :func:`glob.glob` patterns can be used as facet values to select + multiple datasets. + If for some of the datasets not all glob patterns can be expanded + (e.g. because the required facet values cannot be inferred from the + directory names), these datasets will be ignored, unless this happens + to be all datasets. + + If :func:`glob.glob` patterns are used in supplementary variables and + multiple matching datasets are found, only the supplementary dataset + that has most facets in common with the main dataset will be attached. + + Supplementary datasets will in inherit the facet values from the main + dataset for those facets listed in :obj:`INHERITED_FACETS`. + + Examples + -------- + See :ref:`/notebooks/discovering-data.ipynb` for example use cases. + Yields ------ Dataset @@ -133,48 +249,126 @@ def from_files(self) -> Iterator['Dataset']: """ expanded = False if any(_isglob(v) for v in self.facets.values()): - for facets in self._get_available_facets(): - updated_facets = { - k: v - for k, v in facets.items() - if k in self.facets and _isglob(self.facets[k]) - and _ismatch(v, self.facets[k]) - } - dataset = self.copy() - dataset.facets.update(updated_facets) - dataset._update_timerange() - - ancillaries: list['Dataset'] = [] - for ancillary_ds in dataset.ancillaries: - afacets = ancillary_ds.facets - for key, value in updated_facets.items(): - if _isglob(afacets.get(key)): - # Only overwrite ancillary facets that were globs. - afacets[key] = value - ancillaries.extend(ancillary_ds.from_files()) - dataset.ancillaries = ancillaries - - expanded = True - yield dataset + if _isglob(self.facets['mip']): + available_mips = _get_mips( + self.facets['project'], # type: ignore + self.facets['short_name'], # type: ignore + ) + mips = [ + mip for mip in available_mips + if _ismatch(mip, self.facets['mip']) + ] + else: + mips = [self.facets['mip']] # type: ignore + + for mip in mips: + dataset_template = self.copy(mip=mip) + for dataset in dataset_template._get_available_datasets(): + expanded = True + yield dataset if not expanded: - # If the definition contains no wildcards or no files were found, - # yield the original (but do expand any ancillary globs). - ancillaries = [] - for ancillary_ds in self.ancillaries: - ancillaries.extend(ancillary_ds.from_files()) - self.ancillaries = ancillaries + # If the definition contains no wildcards, no files were found, + # or the file facets didn't match the specification, yield the + # original, but do expand any supplementary globs. + self._supplementaries_from_files() yield self + def _supplementaries_from_files(self) -> None: + """Expand wildcards in supplementary datasets.""" + supplementaries: list[Dataset] = [] + for supplementary_ds in self.supplementaries: + for facet in INHERITED_FACETS: + if facet in self.facets: + supplementary_ds.facets[facet] = self.facets[facet] + supplementaries.extend(supplementary_ds.from_files()) + self.supplementaries = supplementaries + self._remove_unexpanded_supplementaries() + self._remove_duplicate_supplementaries() + self._fix_fx_exp() + + def _remove_unexpanded_supplementaries(self) -> None: + """Remove supplementaries where wildcards could not be expanded.""" + supplementaries = [] + for supplementary_ds in self.supplementaries: + unexpanded = [ + f for f, v in supplementary_ds.facets.items() if _isglob(v) + ] + if unexpanded: + logger.info( + "For %s: ignoring supplementary variable '%s', " + "unable to expand wildcards %s.", + self.summary(shorten=True), + supplementary_ds.facets['short_name'], + ", ".join(f"'{f}'" for f in unexpanded), + ) + else: + supplementaries.append(supplementary_ds) + self.supplementaries = supplementaries + + def _match(self, other: Dataset) -> int: + """Compute the match between two datasets.""" + score = 0 + for facet, value2 in self.facets.items(): + if facet in other.facets: + value1 = other.facets[facet] + if isinstance(value1, (list, tuple)): + if isinstance(value2, (list, tuple)): + score += any(elem in value2 for elem in value1) + else: + score += value2 in value1 + else: + if isinstance(value2, (list, tuple)): + score += value1 in value2 + else: + score += value1 == value2 + return score + + def _remove_duplicate_supplementaries(self) -> None: + """Remove supplementaries that are duplicates.""" + not_used = [] + supplementaries = list(self.supplementaries) + self.supplementaries.clear() + for _, duplicates in groupby(supplementaries, + key=lambda ds: ds['short_name']): + group = sorted(duplicates, key=self._match, reverse=True) + self.supplementaries.append(group[0]) + not_used.extend(group[1:]) + + if not_used: + logger.debug( + "List of all supplementary datasets found for %s:\n%s", + self.summary(shorten=True), + "\n".join( + sorted(ds.summary(shorten=True) + for ds in supplementaries)), + ) + + def _fix_fx_exp(self) -> None: + for supplementary_ds in self.supplementaries: + exps = supplementary_ds.facets.get('exp') + frequency = supplementary_ds.facets.get('frequency') + if isinstance(exps, list) and len(exps) > 1 and frequency == 'fx': + for exp in exps: + dataset = supplementary_ds.copy(exp=exp) + if dataset.files: + supplementary_ds.facets['exp'] = exp + logger.info( + "Corrected wrong 'exp' from '%s' to '%s' for " + "supplementary variable '%s' of %s", exps, exp, + supplementary_ds.facets['short_name'], + self.summary(shorten=True)) + break + def copy(self, **facets: FacetValue) -> 'Dataset': """Create a copy. Parameters ---------- **facets - Update these facets in the copy. Note that for ancillary datasets - attached to the dataset, the ``'short_name'`` and ``'mip'`` facets - will not be updated with these values. + Update these facets in the copy. Note that for supplementary + datasets attached to the dataset, the ``'short_name'`` and + ``'mip'`` facets will not be updated with these values. Returns ------- @@ -187,16 +381,16 @@ def copy(self, **facets: FacetValue) -> 'Dataset': new.set_facet(key, deepcopy(value), key in self._persist) for key, value in facets.items(): new.set_facet(key, deepcopy(value)) - for ancillary in self.ancillaries: - # The short_name and mip of the ancillary variable are probably + for supplementary in self.supplementaries: + # The short_name and mip of the supplementary variable are probably # different from the main variable, so don't copy those facets. skip = ('short_name', 'mip') - ancillary_facets = { + supplementary_facets = { k: v for k, v in facets.items() if k not in skip } - new_ancillary = ancillary.copy(**ancillary_facets) - new.ancillaries.append(new_ancillary) + new_supplementary = supplementary.copy(**supplementary_facets) + new.supplementaries.append(new_supplementary) return new def __eq__(self, other) -> bool: @@ -204,7 +398,7 @@ def __eq__(self, other) -> bool: return (isinstance(other, self.__class__) and self._session == other._session and self.facets == other.facets - and self.ancillaries == other.ancillaries) + and self.supplementaries == other.supplementaries) def __repr__(self) -> str: """Create a string representation.""" @@ -230,11 +424,11 @@ def facets2str(facets): f"{self.__class__.__name__}:", facets2str(self.facets), ] - if self.ancillaries: - txt.append("ancillaries:") + if self.supplementaries: + txt.append("supplementaries:") txt.extend( textwrap.indent(facets2str(a.facets), " ") - for a in self.ancillaries) + for a in self.supplementaries) if self._session: txt.append(f"session: '{self.session.session_name}'") return "\n".join(txt) @@ -274,15 +468,14 @@ def summary(self, shorten: bool = False) -> str: f"{title}: " + ", ".join(str(self.facets[k]) for k in keys if k in self.facets)) - def ancillary_summary(dataset): + def supplementary_summary(dataset): return ", ".join( str(dataset.facets[k]) for k in keys if k in dataset.facets and dataset[k] != self.facets.get(k)) - if self.ancillaries: - txt += (", ancillaries: " + - "; ".join(ancillary_summary(a) - for a in self.ancillaries) + "") + if self.supplementaries: + txt += (", supplementaries: " + "; ".join( + supplementary_summary(a) for a in self.supplementaries) + "") return txt def __getitem__(self, key): @@ -324,8 +517,8 @@ def set_version(self) -> None: version = versions.pop() if len(versions) == 1 else sorted(versions) if version: self.set_facet('version', version) - for ancillary_ds in self.ancillaries: - ancillary_ds.set_version() + for supplementary_ds in self.supplementaries: + supplementary_ds.set_version() @property def session(self) -> Session: @@ -338,27 +531,27 @@ def session(self) -> Session: @session.setter def session(self, session: Session | None) -> None: self._session = session - for ancillary in self.ancillaries: - ancillary._session = session + for supplementary in self.supplementaries: + supplementary._session = session - def add_ancillary(self, **facets: FacetValue) -> None: - """Add an ancillary dataset. + def add_supplementary(self, **facets: FacetValue) -> None: + """Add an supplementary dataset. This is a convenience function that will create a copy of the current dataset, update its facets with the values specified in ``**facets``, - and append it to :obj:`Dataset.ancillaries`. For more control - over the creation of the ancillary dataset, first create a new - :class:`Dataset` describing the ancillary dataset and then append - it to :obj:`Dataset.ancillaries`. + and append it to :obj:`Dataset.supplementaries`. For more control + over the creation of the supplementary dataset, first create a new + :class:`Dataset` describing the supplementary dataset and then append + it to :obj:`Dataset.supplementaries`. Parameters ---------- **facets - Facets describing the ancillary variable. + Facets describing the supplementary variable. """ - ancillary = self.copy(**facets) - ancillary.ancillaries = [] - self.ancillaries.append(ancillary) + supplementary = self.copy(**facets) + supplementary.supplementaries = [] + self.supplementaries.append(supplementary) def augment_facets(self) -> None: """Add extra facets. @@ -367,17 +560,11 @@ def augment_facets(self) -> None: from various sources. """ self._augment_facets() - for ancillary in self.ancillaries: - ancillary._augment_facets() + for supplementary in self.supplementaries: + supplementary._augment_facets() def _augment_facets(self): - extra_facets = get_extra_facets( - project=self.facets['project'], - dataset=self.facets['dataset'], - mip=self.facets['mip'], - short_name=self.facets['short_name'], - extra_facets_dir=self.session['extra_facets_dir'], - ) + extra_facets = get_extra_facets(self, self.session['extra_facets_dir']) _augment(self.facets, extra_facets) if 'institute' not in self.facets: institute = get_institutes(self.facets) @@ -395,7 +582,7 @@ def find_files(self) -> None: """Find files. Look for files and populate the :obj:`Dataset.files` property of - the dataset and its ancillary datasets. + the dataset and its supplementary datasets. """ self.augment_facets() @@ -403,8 +590,8 @@ def find_files(self) -> None: self._update_timerange() self._find_files() - for ancillary in self.ancillaries: - ancillary.find_files() + for supplementary in self.supplementaries: + supplementary.find_files() def _find_files(self) -> None: self.files, self._file_globs = local.find_files( @@ -418,12 +605,7 @@ def _find_files(self) -> None: and project in esgf.facets.FACETS) if search_esgf and not self.session['always_search_esgf']: try: - check.data_availability( - self.files, - self.facets, - self._file_globs, - log=False, - ) + check.data_availability(self, log=False) except InputFilesNotFound: pass else: @@ -470,24 +652,34 @@ def load(self) -> Cube: iris.cube.Cube An :mod:`iris` cube with the data corresponding the the dataset. """ - cube = self._load() + return self._load_with_callback(callback='default') + def _load_with_callback(self, callback): + # TODO: Remove the callback argument for v2.10.0. input_files = list(self.files) - ancillary_cubes = [] - for ancillary_dataset in self.ancillaries: - input_files.extend(ancillary_dataset.files) - ancillary_cube = ancillary_dataset._load() - ancillary_cubes.append(ancillary_cube) + for supplementary_dataset in self.supplementaries: + input_files.extend(supplementary_dataset.files) + esgf.download(input_files, self.session['download_dir']) + + cube = self._load(callback) + supplementary_cubes = [] + for supplementary_dataset in self.supplementaries: + supplementary_cube = supplementary_dataset._load(callback) + supplementary_cubes.append(supplementary_cube) + output_file = _get_output_file(self.facets, self.session.preproc_dir) cubes = preprocess( [cube], - 'add_ancillary_variables', + 'add_supplementary_variables', input_files=input_files, - ancillary_cubes=ancillary_cubes, + output_file=output_file, + debug=self.session['save_intermediary_cubes'], + supplementary_cubes=supplementary_cubes, ) + return cubes[0] - def _load(self) -> Cube: + def _load(self, callback) -> Cube: """Load self.files into an iris cube and return it.""" if not self.files: lines = [ @@ -507,7 +699,7 @@ def _load(self) -> Cube: 'output_dir': Path(f"{output_file.with_suffix('')}_fixed"), **self.facets, } - settings['load'] = {} + settings['load'] = {'callback': callback} settings['fix_metadata'] = { 'check_level': self.session['check_level'], **self.facets, @@ -536,9 +728,20 @@ def _load(self) -> Cube: 'short_name': self.facets['short_name'], } - result = self.files + result = [ + file.local_file(self.session['download_dir']) if isinstance( + file, esgf.ESGFFile) else file for file in self.files + ] for step, kwargs in settings.items(): - result = preprocess(result, step, input_files=self.files, **kwargs) + result = preprocess( + result, + step, + input_files=self.files, + output_file=output_file, + debug=self.session['save_intermediary_cubes'], + **kwargs, + ) + cube = result[0] return cube @@ -603,7 +806,11 @@ def _update_timerange(self): If the timerange is given as a year, it ensures it's formatted as a 4-digit value (YYYY). """ - if 'timerange' not in self.facets: + dataset = self.copy() + dataset.supplementaries = [] + dataset.augment_facets() + if 'timerange' not in dataset.facets: + self.facets.pop('timerange', None) return timerange = self.facets['timerange'] @@ -615,12 +822,8 @@ def _update_timerange(self): if '*' in timerange: dataset = self.copy() dataset.facets.pop('timerange') - dataset.ancillaries = [] - check.data_availability( - dataset.files, - dataset.facets, - dataset._file_globs, - ) + dataset.supplementaries = [] + check.data_availability(dataset) intervals = [_get_start_end_date(f.name) for f in dataset.files] min_date = min(interval[0] for interval in intervals) diff --git a/esmvalcore/esgf/_download.py b/esmvalcore/esgf/_download.py index f790d7d784..7efe389293 100644 --- a/esmvalcore/esgf/_download.py +++ b/esmvalcore/esgf/_download.py @@ -301,6 +301,7 @@ def _get_facets_from_dataset_id(results) -> Facets: template = results[0].json['dataset_id_template_'][0] keys = re.findall(r"%\((.*?)\)s", template) reverse_facet_map = {v: k for k, v in FACETS[project].items()} + reverse_facet_map['realm'] = 'modeling_realm' reverse_facet_map['mip_era'] = 'project' # CMIP6 oddity reverse_facet_map['variable_id'] = 'short_name' # CMIP6 oddity reverse_facet_map['valid_institute'] = 'institute' # CMIP5 oddity @@ -526,9 +527,13 @@ def download(files, dest_folder, n_jobs=4): DownloadError: Raised if one or more files failed to download. """ + files = [ + file for file in files if isinstance(file, ESGFFile) + and not file.local_file(dest_folder).exists() + ] if not files: - logger.info("All required data is available locally," - " not downloading anything.") + logger.debug("All required data is available locally," + " not downloading anything.") return files = sorted(files) diff --git a/esmvalcore/esgf/_search.py b/esmvalcore/esgf/_search.py index c36b4da2da..76bcfb8c4e 100644 --- a/esmvalcore/esgf/_search.py +++ b/esmvalcore/esgf/_search.py @@ -9,8 +9,8 @@ from ..config._esgf_pyclient import get_esgf_config from ..local import ( _get_start_end_date, - _get_timerange_from_years, _parse_period, + _replace_years_with_timerange, _truncate_dates, ) from ._download import ESGFFile @@ -359,7 +359,7 @@ def cached_search(**facets): if 'version' not in facets or facets['version'] != '*': files = select_latest_versions(files, facets.get('version')) - _get_timerange_from_years(facets) + _replace_years_with_timerange(facets) if 'timerange' in facets: files = select_by_time(files, facets['timerange']) logger.debug("Selected files:\n%s", '\n'.join(str(f) for f in files)) diff --git a/esmvalcore/esgf/facets.py b/esmvalcore/esgf/facets.py index 4a1ee1e987..3947ba2a6c 100644 --- a/esmvalcore/esgf/facets.py +++ b/esmvalcore/esgf/facets.py @@ -10,7 +10,6 @@ 'ensemble': 'ensemble', 'exp': 'experiment', 'frequency': 'time_frequency', - 'realm': 'realm', 'short_name': 'variable', }, 'CMIP5': { @@ -21,7 +20,6 @@ 'institute': 'institute', 'mip': 'cmor_table', 'product': 'product', - 'realm': 'realm', 'short_name': 'variable', }, 'CMIP6': { @@ -49,7 +47,6 @@ 'dataset': 'source_id', 'frequency': 'time_frequency', 'institute': 'institute', - 'realm': 'realm', 'short_name': 'variable', } } diff --git a/esmvalcore/exceptions.py b/esmvalcore/exceptions.py index f7d775a032..ef8ebc0136 100644 --- a/esmvalcore/exceptions.py +++ b/esmvalcore/exceptions.py @@ -39,14 +39,10 @@ class RecipeError(Error): """Recipe contains an error.""" def __init__(self, msg): - super().__init__(self) + super().__init__(msg) self.message = msg self.failed_tasks = [] - def __str__(self): - """Return message string.""" - return self.message - class InputFilesNotFound(RecipeError): """Files that are required to run the recipe have not been found.""" diff --git a/esmvalcore/experimental/recipe.py b/esmvalcore/experimental/recipe.py index 1cf2fcd185..f0ffe85fcc 100644 --- a/esmvalcore/experimental/recipe.py +++ b/esmvalcore/experimental/recipe.py @@ -91,12 +91,10 @@ def _load(self, session: Session) -> RecipeEngine: recipe : :obj:`esmvalcore._recipe.Recipe` Return an instance of the Recipe """ - config_user = session.to_config_user() - - logger.info(pprint.pformat(config_user)) + logger.info(pprint.pformat(session)) return RecipeEngine(raw_recipe=self.data, - config_user=config_user, + session=session, recipe_file=self.path) def run( diff --git a/esmvalcore/experimental/recipe_output.py b/esmvalcore/experimental/recipe_output.py index fe3f1d7c13..e1e549e9cb 100644 --- a/esmvalcore/experimental/recipe_output.py +++ b/esmvalcore/experimental/recipe_output.py @@ -1,16 +1,13 @@ """API for handing recipe output.""" import base64 import logging -import warnings from collections.abc import Mapping from pathlib import Path from typing import Optional, Tuple, Type import iris -from esmvalcore.config import Session -from esmvalcore.config._config import TASKSEP - +from ..config._config import TASKSEP from .recipe_info import RecipeInfo from .recipe_metadata import Contributor, Reference from .templates import get_template @@ -121,7 +118,7 @@ class RecipeOutput(Mapping): Dictionary with recipe output grouped by diagnostic. info : RecipeInfo The recipe used to create the output. - session : Session + session : esmvalcore.config.Session The session used to run the recipe. """ @@ -175,10 +172,7 @@ def from_core_recipe_output(cls, recipe_output: dict): """Construct instance from `_recipe.Recipe` output. The core recipe format is not directly compatible with the API. This - constructor does the following: - - 1. Convert `config-user` dict to an instance of :obj:`Session` - 2. Converts the raw recipe dict to :obj:`RecipeInfo` + constructor converts the raw recipe dict to :obj:`RecipeInfo` Parameters ---------- @@ -187,13 +181,9 @@ def from_core_recipe_output(cls, recipe_output: dict): """ task_output = recipe_output['task_output'] recipe_data = recipe_output['recipe_data'] - recipe_config = recipe_output['recipe_config'] + session = recipe_output['session'] recipe_filename = recipe_output['recipe_filename'] - with warnings.catch_warnings(): - # ignore deprecation warning - warnings.simplefilter("ignore") - session = Session.from_config_user(recipe_config) info = RecipeInfo(recipe_data, filename=recipe_filename) info.resolve() diff --git a/esmvalcore/local.py b/esmvalcore/local.py index 5d78045024..d287910377 100644 --- a/esmvalcore/local.py +++ b/esmvalcore/local.py @@ -148,8 +148,8 @@ def _dates_to_timerange(start_date, end_date): return f'{start_date}/{end_date}' -def _get_timerange_from_years(variable): - """Build `timerange` tag from tags `start_year` and `end_year`.""" +def _replace_years_with_timerange(variable): + """Set `timerange` tag from tags `start_year` and `end_year`.""" start_year = variable.get('start_year') end_year = variable.get('end_year') if start_year and end_year: @@ -439,7 +439,10 @@ def _get_input_filelist(variable): variable['short_name'] = variable['original_short_name'] globs = _get_globs(variable) - logger.debug("Looking for files matching %s", globs) + logger.debug( + "Looking for files matching:\n%s", + "\n".join(str(g) for g in globs), + ) files = list(Path(file) for glob_ in globs for file in glob(str(glob_))) files.sort() # sorting makes it easier to see what was found diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 2543a96648..34be8289fb 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -1,9 +1,12 @@ """Preprocessor module.""" +from __future__ import annotations + import copy import inspect import logging from pathlib import Path from pprint import pformat +from typing import Any, Iterable from iris.cube import Cube @@ -11,11 +14,6 @@ from .._task import BaseTask from ..cmor.check import cmor_check_data, cmor_check_metadata from ..cmor.fix import fix_data, fix_file, fix_metadata -from ._ancillary_vars import ( - add_ancillary_variables, - add_fx_variables, - remove_fx_variables, -) from ._area import ( area_statistics, extract_named_regions, @@ -57,6 +55,12 @@ regrid, ) from ._rolling_window import rolling_window_statistics +from ._supplementary_vars import ( + add_fx_variables, + add_supplementary_variables, + remove_fx_variables, + remove_supplementary_variables, +) from ._time import ( annual_statistics, anomalies, @@ -94,8 +98,6 @@ 'fix_file', # Load cubes from file 'load', - # Derive variable - 'derive', # Metadata reformatting/CMORization 'fix_metadata', # Concatenate all cubes in one @@ -107,9 +109,10 @@ 'fix_data', 'cmor_check_data', # Attach ancillary variables and cell measures - 'add_ancillary_variables', - # Load fx_variables in cube 'add_fx_variables', + 'add_supplementary_variables', + # Derive variable + 'derive', # Time extraction (as defined in the preprocessor section) 'extract_time', 'extract_season', @@ -183,7 +186,8 @@ 'multi_model_statistics', # Bias calculation 'bias', - # Remove fx_variables from cube + # Remove supplementary variables from cube + 'remove_supplementary_variables', 'remove_fx_variables', # Save to file 'save', @@ -211,8 +215,10 @@ """ # The order of initial and final steps cannot be configured -INITIAL_STEPS = DEFAULT_ORDER[:DEFAULT_ORDER.index('add_fx_variables') + 1] -FINAL_STEPS = DEFAULT_ORDER[DEFAULT_ORDER.index('remove_fx_variables'):] +INITIAL_STEPS = DEFAULT_ORDER[:DEFAULT_ORDER.index( + 'add_supplementary_variables') + 1] +FINAL_STEPS = DEFAULT_ORDER[DEFAULT_ORDER.index( + 'remove_supplementary_variables'):] MULTI_MODEL_FUNCTIONS = { 'bias', @@ -351,7 +357,14 @@ def _run_preproc_function(function, items, kwargs, input_files=None): raise -def preprocess(items, step, input_files=None, **settings): +def preprocess( + items, + step, + input_files=None, + output_file=None, + debug=False, + **settings +): """Run preprocessor.""" logger.debug("Running preprocessor step %s", step) function = globals()[step] @@ -373,6 +386,12 @@ def preprocess(items, step, input_files=None, **settings): else: items.extend(item) + if debug: + logger.debug("Result %s", items) + if all(isinstance(elem, Cube) for elem in items): + filename = _get_debug_filename(output_file, step) + save(items, filename) + return items @@ -380,7 +399,7 @@ def get_step_blocks(steps, order): """Group steps into execution blocks.""" blocks = [] prev_step_type = None - for step in order[order.index('load') + 1:order.index('save')]: + for step in order[len(INITIAL_STEPS):-len(FINAL_STEPS)]: if step in steps: step_type = step in MULTI_MODEL_FUNCTIONS if step_type is not prev_step_type: @@ -394,34 +413,55 @@ def get_step_blocks(steps, order): class PreprocessorFile(TrackedFile): """Preprocessor output file.""" - def __init__(self, attributes, settings, ancestors=None): - super().__init__(attributes['filename'], attributes, ancestors) + def __init__( + self, + filename: Path, + attributes: dict[str, Any] | None = None, + settings: dict[str, Any] | None = None, + datasets: list | None = None, + ): + if datasets is not None: + # Load data using a Dataset + input_files = [] + for dataset in datasets: + input_files.extend(dataset.files) + for supplementary in dataset.supplementaries: + input_files.extend(supplementary.files) + ancestors = [TrackedFile(f) for f in input_files] + else: + # Multimodel preprocessor functions set ancestors at runtime + # instead of here. + input_files = [] + ancestors = [] + self.datasets = datasets + self._cubes = None + self._input_files = input_files + + # Set some preprocessor settings (move all defaults here?) + if settings is None: + settings = {} self.settings = copy.deepcopy(settings) + if attributes is None: + attributes = {} + attributes = copy.deepcopy(attributes) if 'save' not in self.settings: self.settings['save'] = {} - self.settings['save']['filename'] = self.filename + self.settings['save']['filename'] = filename - # self._input_files always contains the original input files; - # self.files may change in the preprocessing chain (e.g., by the step - # fix_file) - self._input_files = [a.filename for a in ancestors or ()] - self.files = copy.deepcopy(self._input_files) + attributes['filename'] = filename - self._cubes = None - self._prepared = False - - def _input_files_for_log(self): - """Do not log input files twice in output log.""" - if self.files == self._input_files: - return None - return self._input_files + super().__init__( + filename=filename, + attributes=attributes, + ancestors=ancestors, + ) def check(self): """Check preprocessor settings.""" check_preprocessor_settings(self.settings) - def apply(self, step, debug=False): + def apply(self, step: str, debug: bool = False): """Apply preprocessor step to product.""" if step not in self.settings: raise ValueError( @@ -429,31 +469,18 @@ def apply(self, step, debug=False): ) self.cubes = preprocess(self.cubes, step, input_files=self._input_files, + output_file=self.filename, + debug=debug, **self.settings[step]) - if debug: - logger.debug("Result %s", self.cubes) - filename = _get_debug_filename(self.filename, step) - save(self.cubes, filename) - - def prepare(self): - """Apply preliminary file operations on product.""" - if not self._prepared: - for step in DEFAULT_ORDER[:DEFAULT_ORDER.index('load')]: - if step in self.settings: - self.files = preprocess( - self.files, step, - input_files=self._input_files_for_log(), - **self.settings[step]) - self._prepared = True @property def cubes(self): """Cubes.""" - if self.is_closed: - self.prepare() - self._cubes = preprocess(self.files, 'load', - input_files=self._input_files_for_log(), - **self.settings.get('load', {})) + if self._cubes is None: + callback = self.settings.get('load', {}).get('callback') + self._cubes = [ + ds._load_with_callback(callback) for ds in self.datasets + ] return self._cubes @cubes.setter @@ -462,12 +489,14 @@ def cubes(self, value): def save(self): """Save cubes to disk.""" - self.files = preprocess(self._cubes, 'save', - input_files=self._input_files, - **self.settings['save']) - self.files = preprocess(self.files, 'cleanup', - input_files=self._input_files, - **self.settings.get('cleanup', {})) + preprocess(self._cubes, + 'save', + input_files=self._input_files, + **self.settings['save']) + preprocess([], + 'cleanup', + input_files=self._input_files, + **self.settings.get('cleanup', {})) def close(self): """Close the file.""" @@ -506,7 +535,7 @@ def is_closed(self): return self._cubes is None def _initialize_entity(self): - """Initialize the entity representing the file.""" + """Initialize the provenance entity representing the file.""" super()._initialize_entity() settings = { 'preprocessor:' + k: str(v) @@ -537,10 +566,6 @@ def group(self, keys: list) -> str: return '_'.join(identifier) -# TODO: use a custom ProductSet that raises an exception if you try to -# add the same Product twice - - def _apply_multimodel(products, step, debug): """Apply multi model step to products.""" settings, exclude = _get_multi_model_settings(products, step) @@ -565,16 +590,15 @@ class PreprocessingTask(BaseTask): def __init__( self, - products, - ancestors=None, - name='', - order=DEFAULT_ORDER, - debug=None, - write_ncl_interface=False, + products: Iterable[PreprocessorFile], + name: str = '', + order: Iterable[str] = DEFAULT_ORDER, + debug: bool | None = None, + write_ncl_interface: bool = False, ): """Initialize.""" _check_multi_model_settings(products) - super().__init__(ancestors=ancestors, name=name, products=products) + super().__init__(name=name, products=products) self.order = list(order) self.debug = debug self.write_ncl_interface = write_ncl_interface @@ -629,6 +653,12 @@ def _run(self, _): for product in self.products for step in product.settings } blocks = get_step_blocks(steps, self.order) + if not blocks: + # If no preprocessing is configured, just load the data and save. + for product in self.products: + product.cubes # pylint: disable=pointless-statement + product.close() + for block in blocks: logger.debug("Running block %s", block) if block[0] in MULTI_MODEL_FUNCTIONS: @@ -642,6 +672,7 @@ def _run(self, _): if step in product.settings: product.apply(step, self.debug) if block == blocks[-1]: + product.cubes # pylint: disable=pointless-statement product.close() for product in self.products: @@ -656,8 +687,11 @@ def __str__(self): step for step in self.order if any(step in product.settings for product in self.products) ] - products = '\n\n'.join('\n'.join([str(p), pformat(p.settings)]) - for p in self.products) + products = '\n\n'.join('\n'.join([ + str(p), + 'input files: ' + pformat(p._input_files), + 'settings: ' + pformat(p.settings), + ]) for p in self.products) txt = "\n".join([ f"{self.__class__.__name__}: {self.name}", f"order: {order}", diff --git a/esmvalcore/preprocessor/_area.py b/esmvalcore/preprocessor/_area.py index a982367649..5090fdf209 100644 --- a/esmvalcore/preprocessor/_area.py +++ b/esmvalcore/preprocessor/_area.py @@ -14,16 +14,17 @@ from dask import array as da from iris.exceptions import CoordinateNotFoundError -from ._ancillary_vars import ( - add_ancillary_variable, - add_cell_measure, - remove_fx_variables, -) from ._shared import ( get_iris_analysis_operation, guess_bounds, operator_accept_weights, ) +from ._supplementary_vars import ( + add_ancillary_variable, + add_cell_measure, + register_supplementaries, + remove_supplementary_variables, +) logger = logging.getLogger(__name__) @@ -202,6 +203,10 @@ def compute_area_weights(cube): return weights +@register_supplementaries( + variables=['areacella', 'areacello'], + required='prefer_at_least_one', +) def area_statistics(cube, operator): """Apply a statistical operator in the horizontal direction. @@ -235,7 +240,11 @@ def area_statistics(cube, operator): Parameters ---------- cube: iris.cube.Cube - Input cube. + Input cube. The input cube should have a + :class:`iris.coords.CellMeasure` named ``'cell_area'``, unless it + has regular 1D latitude and longitude coordinates so the cell areas + can be computed using + :func:`iris.analysis.cartography.area_weights`. operator: str The operation, options: mean, median, min, max, std_dev, sum, variance, rms. @@ -656,7 +665,7 @@ def _mask_cube(cube, selections): cubelist = iris.cube.CubeList() for id_, select in selections.items(): _cube = cube.copy() - remove_fx_variables(_cube) + remove_supplementary_variables(_cube) _cube.add_aux_coord( iris.coords.AuxCoord(id_, units='no_unit', long_name="shape_id")) select = da.broadcast_to(select, _cube.shape) diff --git a/esmvalcore/preprocessor/_derive/xco2.py b/esmvalcore/preprocessor/_derive/xco2.py index 69204edc13..d341002c7f 100644 --- a/esmvalcore/preprocessor/_derive/xco2.py +++ b/esmvalcore/preprocessor/_derive/xco2.py @@ -25,7 +25,6 @@ def calculate(cubes): """Calculate the column-averaged atmospheric CO2 [1e-6].""" co2_cube = cubes.extract_cube( Constraint(name='mole_fraction_of_carbon_dioxide_in_air')) - print(co2_cube) hus_cube = cubes.extract_cube(Constraint(name='specific_humidity')) zg_cube = cubes.extract_cube(Constraint(name='geopotential_height')) ps_cube = cubes.extract_cube(Constraint(name='surface_air_pressure')) diff --git a/esmvalcore/preprocessor/_io.py b/esmvalcore/preprocessor/_io.py index 467e75e78d..ba134968f8 100644 --- a/esmvalcore/preprocessor/_io.py +++ b/esmvalcore/preprocessor/_io.py @@ -3,6 +3,7 @@ import logging import os import shutil +import warnings from itertools import groupby from warnings import catch_warnings, filterwarnings @@ -12,6 +13,7 @@ import yaml from cf_units import suppress_errors +from esmvalcore.exceptions import ESMValCoreDeprecationWarning from esmvalcore.iris_helpers import merge_cube_attributes from .._task import write_ncl_settings @@ -121,6 +123,9 @@ def load(file, callback=None, ignore_warnings=None): File to be loaded. callback: callable or None, optional (default: None) Callback function passed to :func:`iris.load_raw`. + + .. deprecated:: 2.8.0 + This argument will be removed in 2.10.0. ignore_warnings: list of dict or None, optional (default: None) Keyword arguments passed to :func:`warnings.filterwarnings` used to ignore warnings issued by :func:`iris.load_raw`. Each list element @@ -136,6 +141,13 @@ def load(file, callback=None, ignore_warnings=None): ValueError Cubes are empty. """ + if not (callback is None or callback == 'default'): + msg = ("The argument `callback` has been deprecated in " + "ESMValCore version 2.8.0 and is scheduled for removal in " + "version 2.10.0.") + warnings.warn(msg, ESMValCoreDeprecationWarning) + if callback == 'default': + callback = concatenate_callback file = str(file) logger.debug("Loading:\n%s", file) if ignore_warnings is None: diff --git a/esmvalcore/preprocessor/_mask.py b/esmvalcore/preprocessor/_mask.py index 8cf983aa3d..fa411bdd0d 100644 --- a/esmvalcore/preprocessor/_mask.py +++ b/esmvalcore/preprocessor/_mask.py @@ -16,6 +16,8 @@ from iris.analysis import Aggregator from iris.util import rolling_window +from ._supplementary_vars import register_supplementaries + logger = logging.getLogger(__name__) @@ -59,20 +61,30 @@ def _apply_fx_mask(fx_mask, var_data): return var_data +@register_supplementaries( + variables=['sftlf', 'sftof'], + required='prefer_at_least_one', +) def mask_landsea(cube, mask_out): """Mask out either land mass or sea (oceans, seas and lakes). It uses dedicated ancillary variables (sftlf or sftof) or, in their absence, it applies a - Natural Earth mask (land or ocean contours). + `Natural Earth `_ mask (land or ocean + contours). Note that the Natural Earth masks have different resolutions: 10m for land, and 50m for seas. - These are more than enough for ESMValTool purposes. + These are more than enough for masking climate model data. Parameters ---------- cube: iris.cube.Cube - data cube to be masked. + data cube to be masked. If the cube has an + :class:`iris.coords.AncillaryVariable` with standard name + ``'land_area_fraction'`` or ``'sea_area_fraction'`` that will be used. + If both are present, only the 'land_area_fraction' will be used. If the + ancillary variable is not available, the mask will be calculated from + Natural Earth shapefiles. mask_out: str either "land" to mask out land mass or "sea" to mask out seas. @@ -85,7 +97,8 @@ def mask_landsea(cube, mask_out): Raises ------ ValueError - Error raised if masking on irregular grids is attempted. + Error raised if masking on irregular grids is attempted without + an ancillary variable. Irregular grids are not currently supported for masking with Natural Earth shapefile masks. """ @@ -131,6 +144,10 @@ def mask_landsea(cube, mask_out): return cube +@register_supplementaries( + variables=['sftgif'], + required='require_at_least_one', +) def mask_landseaice(cube, mask_out): """Mask out either landsea (combined) or ice. @@ -142,7 +159,9 @@ def mask_landseaice(cube, mask_out): Parameters ---------- cube: iris.cube.Cube - data cube to be masked. + data cube to be masked. It should have an + :class:`iris.coords.AncillaryVariable` with standard name + ``'land_ice_area_fraction'``. mask_out: str either "landsea" to mask out landsea or "ice" to mask out ice. @@ -176,7 +195,7 @@ def mask_landseaice(cube, mask_out): return cube -def mask_glaciated(cube, mask_out): +def mask_glaciated(cube, mask_out: str = "glaciated"): """Mask out glaciated areas. It applies a Natural Earth mask. Note that for computational reasons diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 89f1357238..f808d1d9c5 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -23,7 +23,9 @@ from iris.util import equalise_attributes, new_axis from esmvalcore.iris_helpers import date2num -from esmvalcore.preprocessor import remove_fx_variables +from esmvalcore.preprocessor._supplementary_vars import ( + remove_supplementary_variables, +) from ._other import _group_products @@ -371,7 +373,7 @@ def _equalise_fx_variables(cubes): """Equalise fx variables in cubes (in-place).""" # Simple remove all fx variables for cube in cubes: - remove_fx_variables(cube) + remove_supplementary_variables(cube) def _equalise_var_metadata(cubes): diff --git a/esmvalcore/preprocessor/_regrid.py b/esmvalcore/preprocessor/_regrid.py index dcddd2af1b..302e0c86be 100644 --- a/esmvalcore/preprocessor/_regrid.py +++ b/esmvalcore/preprocessor/_regrid.py @@ -19,12 +19,10 @@ from iris.util import broadcast_to_shape from ..cmor._fixes.shared import add_altitude_from_plev, add_plev_from_altitude -from ..cmor.fix import fix_file, fix_metadata from ..cmor.table import CMOR_TABLES -from ._ancillary_vars import add_ancillary_variable, add_cell_measure -from ._io import concatenate_callback, load from ._regrid_esmpy import ESMF_REGRID_METHODS from ._regrid_esmpy import regrid as esmpy_regrid +from ._supplementary_vars import add_ancillary_variable, add_cell_measure logger = logging.getLogger(__name__) @@ -453,6 +451,12 @@ def extract_point(cube, latitude, longitude, scheme): return cube +def is_dataset(dataset): + """Test if something is an `esmvalcore.dataset.Dataset`.""" + # Use this function to avoid circular imports + return hasattr(dataset, 'facets') + + def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): """Perform horizontal regridding. @@ -544,25 +548,28 @@ def regrid(cube, target_grid, scheme, lat_offset=True, lon_offset=True): target: 1x1 scheme: reference: esmf_regrid.schemes:ESMFAreaWeighted - """ - if isinstance(target_grid, (str, Path)): - if os.path.isfile(target_grid): - target_grid = iris.load_cube(target_grid) - else: - # Generate a target grid from the provided cell-specification, - # and cache the resulting stock cube for later use. - target_grid = _CACHE.setdefault( - target_grid, - _global_stock_cube(target_grid, lat_offset, lon_offset), - ) - # Align the target grid coordinate system to the source - # coordinate system. - src_cs = cube.coord_system() - xcoord = target_grid.coord(axis='x', dim_coords=True) - ycoord = target_grid.coord(axis='y', dim_coords=True) - xcoord.coord_system = src_cs - ycoord.coord_system = src_cs + if is_dataset(target_grid): + target_grid = target_grid.copy() + target_grid.supplementaries.clear() + target_grid.files = [target_grid.files[0]] + target_grid = target_grid.load() + elif isinstance(target_grid, (str, Path)) and os.path.isfile(target_grid): + target_grid = iris.load_cube(target_grid) + elif isinstance(target_grid, str): + # Generate a target grid from the provided cell-specification, + # and cache the resulting stock cube for later use. + target_grid = _CACHE.setdefault( + target_grid, + _global_stock_cube(target_grid, lat_offset, lon_offset), + ) + # Align the target grid coordinate system to the source + # coordinate system. + src_cs = cube.coord_system() + xcoord = target_grid.coord(axis='x', dim_coords=True) + ycoord = target_grid.coord(axis='y', dim_coords=True) + xcoord.coord_system = src_cs + ycoord.coord_system = src_cs elif isinstance(target_grid, dict): # Generate a target grid from the provided specification, target_grid = _regional_stock_cube(target_grid) @@ -1012,26 +1019,13 @@ def get_cmor_levels(cmor_table, coordinate): ) -def get_reference_levels(filename, project, dataset, short_name, mip, - frequency, fix_dir): +def get_reference_levels(dataset): """Get level definition from a reference dataset. Parameters ---------- - filename: str - Path to the reference file - project : str - Name of the project - dataset : str - Name of the dataset - short_name : str - Name of the variable - mip : str - Name of the mip table - frequency : str - Time frequency - fix_dir : str - Output directory for fixed data + dataset: esmvalcore.dataset.Dataset + Dataset containing the reference files. Returns ------- @@ -1043,28 +1037,14 @@ def get_reference_levels(filename, project, dataset, short_name, mip, If the dataset is not defined, the coordinate does not specify any levels or the string is badly formatted. """ - filename = fix_file( - file=filename, - short_name=short_name, - project=project, - dataset=dataset, - mip=mip, - output_dir=fix_dir, - ) - cubes = load(filename, callback=concatenate_callback) - cubes = fix_metadata( - cubes=cubes, - short_name=short_name, - project=project, - dataset=dataset, - mip=mip, - frequency=frequency, - ) - cube = cubes[0] + dataset = dataset.copy() + dataset.supplementaries.clear() + dataset.files = [dataset.files[0]] + cube = dataset.load() try: coord = cube.coord(axis='Z') - except iris.exceptions.CoordinateNotFoundError: - raise ValueError('z-coord not available in {}'.format(filename)) + except iris.exceptions.CoordinateNotFoundError as exc: + raise ValueError(f'z-coord not available in {dataset.files}') from exc return coord.points.tolist() diff --git a/esmvalcore/preprocessor/_ancillary_vars.py b/esmvalcore/preprocessor/_supplementary_vars.py similarity index 51% rename from esmvalcore/preprocessor/_ancillary_vars.py rename to esmvalcore/preprocessor/_supplementary_vars.py index fb2b5a85a2..96be17d526 100644 --- a/esmvalcore/preprocessor/_ancillary_vars.py +++ b/esmvalcore/preprocessor/_supplementary_vars.py @@ -1,19 +1,53 @@ """Preprocessor functions for ancillary variables and cell measures.""" import logging +import warnings from pathlib import Path from typing import Iterable import dask.array as da -import iris +import iris.coords +import iris.cube from esmvalcore.cmor.check import cmor_check_data, cmor_check_metadata from esmvalcore.cmor.fix import fix_data, fix_metadata -from esmvalcore.preprocessor._io import concatenate, concatenate_callback, load +from esmvalcore.config import CFG +from esmvalcore.exceptions import ESMValCoreDeprecationWarning +from esmvalcore.preprocessor._io import concatenate, load from esmvalcore.preprocessor._time import clip_timerange logger = logging.getLogger(__name__) +PREPROCESSOR_SUPPLEMENTARIES = {} + + +def register_supplementaries(variables, required): + """Register supplementary variables required for a preprocessor function. + + Parameters + ---------- + variables: :obj:`list` of :obj`str` + List of variable names. + required: + How strong the requirement is. Can be 'require_at_least_one' if at + least one variable must be available or 'prefer_at_least_one' if it is + preferred that at least one variable is available, but not strictly + necessary. + """ + valid = ('require_at_least_one', 'prefer_at_least_one') + if required not in valid: + raise NotImplementedError(f"`required` should be one of {valid}") + supplementaries = { + 'variables': variables, + 'required': required, + } + + def wrapper(func): + PREPROCESSOR_SUPPLEMENTARIES[func.__name__] = supplementaries + return func + + return wrapper + def _load_fx(var_cube, fx_info, check_level): """Load and CMOR-check fx variables.""" @@ -25,7 +59,7 @@ def _load_fx(var_cube, fx_info, check_level): freq = fx_info['frequency'] for fx_file in fx_info['filename']: - loaded_cube = load(fx_file, callback=concatenate_callback) + loaded_cube = load(fx_file) loaded_cube = fix_metadata(loaded_cube, check_level=check_level, **fx_info) @@ -70,22 +104,22 @@ def _is_fx_broadcastable(fx_cube, cube): return True -def add_cell_measure(cube, fx_cube, measure): - """Add fx cube as a cell_measure in the cube containing the data. +def add_cell_measure(cube, cell_measure_cube, measure): + """Add a cube as a cell_measure in the cube containing the data. Parameters ---------- cube: iris.cube.Cube Iris cube with input data. - fx_cube: iris.cube.Cube - Iris cube with fx data. + cell_measure_cube: iris.cube.Cube + Iris cube with cell measure data. measure: str Name of the measure, can be 'area' or 'volume'. Returns ------- iris.cube.Cube - Cube with added ancillary variables + Cube with added cell measure Raises ------ @@ -95,27 +129,29 @@ def add_cell_measure(cube, fx_cube, measure): if measure not in ['area', 'volume']: raise ValueError(f"measure name must be 'area' or 'volume', " f"got {measure} instead") - measure = iris.coords.CellMeasure(fx_cube.core_data(), - standard_name=fx_cube.standard_name, - units=fx_cube.units, - measure=measure, - var_name=fx_cube.var_name, - attributes=fx_cube.attributes) + measure = iris.coords.CellMeasure( + cell_measure_cube.core_data(), + standard_name=cell_measure_cube.standard_name, + units=cell_measure_cube.units, + measure=measure, + var_name=cell_measure_cube.var_name, + attributes=cell_measure_cube.attributes, + ) start_dim = cube.ndim - len(measure.shape) cube.add_cell_measure(measure, range(start_dim, cube.ndim)) - logger.debug('Added %s as cell measure in cube of %s.', fx_cube.var_name, - cube.var_name) + logger.debug('Added %s as cell measure in cube of %s.', + cell_measure_cube.var_name, cube.var_name) -def add_ancillary_variable(cube, fx_cube): - """Add fx cube as an ancillary_variable in the cube containing the data. +def add_ancillary_variable(cube, ancillary_cube): + """Add cube as an ancillary variable in the cube containing the data. Parameters ---------- cube: iris.cube.Cube Iris cube with input data. - fx_cube: iris.cube.Cube - Iris cube with fx data. + ancillary_cube: iris.cube.Cube + Iris cube with ancillary data. Returns ------- @@ -123,15 +159,15 @@ def add_ancillary_variable(cube, fx_cube): Cube with added ancillary variables """ ancillary_var = iris.coords.AncillaryVariable( - fx_cube.core_data(), - standard_name=fx_cube.standard_name, - units=fx_cube.units, - var_name=fx_cube.var_name, - attributes=fx_cube.attributes) + ancillary_cube.core_data(), + standard_name=ancillary_cube.standard_name, + units=ancillary_cube.units, + var_name=ancillary_cube.var_name, + attributes=ancillary_cube.attributes) start_dim = cube.ndim - len(ancillary_var.shape) cube.add_ancillary_variable(ancillary_var, range(start_dim, cube.ndim)) logger.debug('Added %s as ancillary variable in cube of %s.', - fx_cube.var_name, cube.var_name) + ancillary_cube.var_name, cube.var_name) def add_fx_variables(cube, fx_variables, check_level): @@ -139,6 +175,12 @@ def add_fx_variables(cube, fx_variables, check_level): variables as cell measures or ancillary variables in the cube containing the data. + .. deprecated:: 2.8.0 + This function is deprecated and will be removed in version 2.10.0. + Please use a :class:`esmvalcore.dataset.Dataset` or + :func:`esmvalcore.preprocessor.add_supplementary_variables` + instead. + Parameters ---------- cube: iris.cube.Cube @@ -148,13 +190,17 @@ def add_fx_variables(cube, fx_variables, check_level): check_level: CheckLevels Level of strictness of the checks. - Returns ------- iris.cube.Cube Cube with added cell measures or ancillary variables. """ - + msg = ( + "The function `add_fx_variables` has been deprecated in " + "ESMValCore version 2.8.0 and is scheduled for removal in " + "version 2.10.0. Use a `esmvalcore.dataset.Dataset` or the function " + "`add_supplementary_variables` instead.") + warnings.warn(msg, ESMValCoreDeprecationWarning) if not fx_variables: return cube fx_cubes = [] @@ -170,13 +216,13 @@ def add_fx_variables(cube, fx_variables, check_level): fx_cubes.append(fx_cube) - add_ancillary_variables(cube, fx_cubes) + add_supplementary_variables(cube, fx_cubes) return cube -def add_ancillary_variables( +def add_supplementary_variables( cube: iris.cube.Cube, - ancillary_cubes: Iterable[iris.cube.Cube], + supplementary_cubes: Iterable[iris.cube.Cube], ) -> iris.cube.Cube: """Add ancillary variables and/or cell measures. @@ -184,8 +230,8 @@ def add_ancillary_variables( ---------- cube: Cube to add to. - ancillary_cubes: - Iterable of cubes containing the ancillary variables. + supplementary_cubes: + Iterable of cubes containing the supplementary variables. Returns ------- @@ -197,35 +243,66 @@ def add_ancillary_variables( 'areacello': 'area', 'volcello': 'volume' } - for ancillary_cube in ancillary_cubes: - if ancillary_cube.var_name in measure_names: - measure_name = measure_names[ancillary_cube.var_name] - add_cell_measure(cube, ancillary_cube, measure_name) + for supplementary_cube in supplementary_cubes: + if (CFG['use_legacy_supplementaries'] + and not _is_fx_broadcastable(supplementary_cube, cube)): + continue + if supplementary_cube.var_name in measure_names: + measure_name = measure_names[supplementary_cube.var_name] + add_cell_measure(cube, supplementary_cube, measure_name) else: - add_ancillary_variable(cube, ancillary_cube) + add_ancillary_variable(cube, supplementary_cube) return cube -def remove_fx_variables(cube): - """Remove fx variables present as cell measures or ancillary variables in - the cube containing the data. +def remove_supplementary_variables(cube: iris.cube.Cube): + """Remove supplementary variables. + + Strip cell measures or ancillary variables from the cube containing the + data. Parameters ---------- cube: iris.cube.Cube - Iris cube with data and cell measures or ancillary variables. - + Iris cube with data and cell measures or ancillary variables. Returns ------- iris.cube.Cube Cube without cell measures or ancillary variables. """ - if cube.cell_measures(): for measure in cube.cell_measures(): - cube.remove_cell_measure(measure.standard_name) + cube.remove_cell_measure(measure) if cube.ancillary_variables(): for variable in cube.ancillary_variables(): - cube.remove_ancillary_variable(variable.standard_name) + cube.remove_ancillary_variable(variable) return cube + + +def remove_fx_variables(cube): + """Remove fx variables present as cell measures or ancillary variables in + the cube containing the data. + + .. deprecated:: 2.8.0 + This function is deprecated and will be removed in version 2.10.0. + Please use + :func:`esmvalcore.preprocessor.remove_supplementary_variables` + instead. + + Parameters + ---------- + cube: iris.cube.Cube + Iris cube with data and cell measures or ancillary variables. + + Returns + ------- + iris.cube.Cube + Cube without cell measures or ancillary variables. + """ + msg = ("The function `remove_fx_variables` has been deprecated in " + "ESMValCore version 2.8.0 and is scheduled for removal in " + "version 2.10.0. Use the function `remove_supplementary_variables` " + "instead.") + warnings.warn(msg, ESMValCoreDeprecationWarning) + return remove_supplementary_variables(cube) diff --git a/esmvalcore/preprocessor/_volume.py b/esmvalcore/preprocessor/_volume.py index 2b93d88c52..654bbcd500 100644 --- a/esmvalcore/preprocessor/_volume.py +++ b/esmvalcore/preprocessor/_volume.py @@ -10,6 +10,7 @@ import numpy as np from ._shared import get_iris_analysis_operation, operator_accept_weights +from ._supplementary_vars import register_supplementaries logger = logging.getLogger(__name__) @@ -56,7 +57,7 @@ def extract_volume(cube, z_min, z_max): def calculate_volume(cube): """Calculate volume from a cube. - This function is used when the volume netcdf fx_variables can't be found. + This function is used when the volume ancillary variables can't be found. Parameters ---------- @@ -88,17 +89,25 @@ def calculate_volume(cube): return grid_volume +@register_supplementaries( + variables=['volcello'], + required='prefer_at_least_one', +) def volume_statistics(cube, operator): """Apply a statistical operation over a volume. - The volume average is weighted according to the cell volume. Cell volume - is calculated from iris's cartography tool multiplied by the cell - thickness. + The volume average is weighted according to the cell volume. Parameters ---------- cube: iris.cube.Cube - Input cube. + Input cube. The input cube should have a + :class:`iris.coords.CellMeasure` with standard name ``'ocean_volume'``, + unless it has regular 1D latitude and longitude coordinates so the cell + volumes can be computed by using + :func:`iris.analysis.cartography.area_weights` to compute the cell + areas and multiplying those by the cell thickness, computed from the + bounds of the vertical coordinate. operator: str The operation to apply to the cube, options are: 'mean'. diff --git a/esmvalcore/preprocessor/_weighting.py b/esmvalcore/preprocessor/_weighting.py index 32e6c526a0..5aa2a70dd5 100644 --- a/esmvalcore/preprocessor/_weighting.py +++ b/esmvalcore/preprocessor/_weighting.py @@ -4,6 +4,8 @@ import iris +from ._supplementary_vars import register_supplementaries + logger = logging.getLogger(__name__) @@ -18,9 +20,8 @@ def _get_land_fraction(cube): try: fx_cube = cube.ancillary_variable('sea_area_fraction') except iris.exceptions.AncillaryVariableNotFoundError: - errors.append( - 'Ancillary variables land/sea area fraction ' - 'not found in cube. Check fx_file availability.') + errors.append('Ancillary variables land/sea area fraction not ' + 'found in cube. Check ancillary data availability.') return (land_fraction, errors) if fx_cube.var_name == 'sftlf': @@ -31,6 +32,10 @@ def _get_land_fraction(cube): return (land_fraction, errors) +@register_supplementaries( + variables=['sftlf', 'sftof'], + required='require_at_least_one', +) def weighting_landsea_fraction(cube, area_type): """Weight fields using land or sea fraction. @@ -45,7 +50,10 @@ def weighting_landsea_fraction(cube, area_type): Parameters ---------- cube : iris.cube.Cube - Data cube to be weighted. + Data cube to be weighted. It should have an + :class:`iris.coords.AncillaryVariable` with standard name + ``'land_area_fraction'`` or ``'sea_area_fraction'``. If both are + present, only the ``'land_area_fraction'`` will be used. area_type : str Use land (``'land'``) or sea (``'sea'``) fraction for weighting. @@ -60,7 +68,6 @@ def weighting_landsea_fraction(cube, area_type): ``area_type`` is not ``'land'`` or ``'sea'``. ValueError Land/sea fraction variables ``sftlf`` or ``sftof`` not found. - """ if area_type not in ('land', 'sea'): raise TypeError( diff --git a/tests/integration/cmor/_fixes/cesm/test_cesm2.py b/tests/integration/cmor/_fixes/cesm/test_cesm2.py index 98f030ccc4..6d15d12e79 100644 --- a/tests/integration/cmor/_fixes/cesm/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cesm/test_cesm2.py @@ -10,6 +10,7 @@ from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info from esmvalcore.config._config import get_extra_facets +from esmvalcore.dataset import Dataset # Note: test_data_path is defined in tests/integration/cmor/_fixes/conftest.py @@ -38,7 +39,13 @@ def cube_1d_time(): def _get_fix(mip, frequency, short_name, fix_name): """Load a fix from :mod:`esmvalcore.cmor._fixes.cesm.cesm2`.""" - extra_facets = get_extra_facets('CESM', 'CESM2', mip, short_name, ()) + dataset = Dataset( + project='CESM', + dataset='CESM2', + mip=mip, + short_name=short_name, + ) + extra_facets = get_extra_facets(dataset, ()) extra_facets['frequency'] = frequency vardef = get_var_info(project='CESM', mip=mip, short_name=short_name) cls = getattr(esmvalcore.cmor._fixes.cesm.cesm2, fix_name) diff --git a/tests/integration/cmor/_fixes/emac/test_emac.py b/tests/integration/cmor/_fixes/emac/test_emac.py index 201b8cd8ef..8b36b7770b 100644 --- a/tests/integration/cmor/_fixes/emac/test_emac.py +++ b/tests/integration/cmor/_fixes/emac/test_emac.py @@ -9,6 +9,7 @@ from iris.coords import AuxCoord, DimCoord from iris.cube import Cube, CubeList +import esmvalcore.cmor._fixes.emac.emac from esmvalcore.cmor._fixes.emac.emac import ( AllVars, Cl, @@ -43,6 +44,7 @@ from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info from esmvalcore.config._config import get_extra_facets +from esmvalcore.dataset import Dataset @pytest.fixture @@ -182,14 +184,39 @@ def cubes_3d(): return cubes -def get_allvars_fix(mip, short_name): - """Get member of fix class.""" - vardef = get_var_info('EMAC', mip, short_name) - extra_facets = get_extra_facets('EMAC', 'EMAC', mip, short_name, ()) - fix = AllVars(vardef, extra_facets=extra_facets) +def _get_fix(mip, short_name, fix_name): + """Load a fix from the esmvalcore.cmor._fixes.emac.emac module.""" + dataset = Dataset( + project='EMAC', + dataset='EMAC', + mip=mip, + short_name=short_name, + ) + extra_facets = get_extra_facets(dataset, ()) + vardef = get_var_info(project='EMAC', mip=mip, short_name=short_name) + cls = getattr(esmvalcore.cmor._fixes.emac.emac, fix_name) + fix = cls(vardef, extra_facets=extra_facets) return fix +def get_fix(mip, short_name): + fix_name = short_name[0].upper() + short_name[1:] + return _get_fix(mip, short_name, fix_name) + + +def get_allvars_fix(mip, short_name): + return _get_fix(mip, short_name, 'AllVars') + + +def fix_metadata(cubes, mip, short_name): + """Fix metadata of cubes.""" + fix = get_fix(mip, short_name) + cubes = fix.fix_metadata(cubes) + fix = get_allvars_fix(mip, short_name) + cubes = fix.fix_metadata(cubes) + return cubes + + def check_tas_metadata(cubes): """Check tas metadata.""" assert len(cubes) == 1 @@ -535,23 +562,25 @@ def test_var_not_available_get_cube(): # Test with single-dimension cubes -def test_only_time(): +def test_only_time(monkeypatch): """Test fix.""" + fix = get_allvars_fix('Amon', 'ta') # We know that ta has dimensions time, plev19, latitude, longitude, but the # EMAC CMORizer is designed to check for the presence of each dimension # individually. To test this, remove all but one dimension of ta to create # an artificial, but realistic test case. - vardef = get_var_info('EMAC', 'Amon', 'ta') - original_dimensions = vardef.dimensions - vardef.dimensions = ['time'] - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'ta', ()) - fix = AllVars(vardef, extra_facets=extra_facets) + monkeypatch.setattr(fix.vardef, 'dimensions', ['time']) # Create cube with only a single dimension - time_coord = DimCoord([0.0, 1.0], var_name='time', standard_name='time', - long_name='time', units='days since 1850-01-01') + time_coord = DimCoord([0.0, 1.0], + var_name='time', + standard_name='time', + long_name='time', + units='days since 1850-01-01') cubes = CubeList([ - Cube([1, 1], var_name='tm1_p19_ave', units='K', + Cube([1, 1], + var_name='tm1_p19_ave', + units='K', dim_coords_and_dims=[(time_coord, 0)]), ]) fixed_cubes = fix.fix_metadata(cubes) @@ -576,27 +605,25 @@ def test_only_time(): np.testing.assert_allclose(new_time_coord.bounds, [[-0.5, 0.5], [0.5, 1.5]]) - # Restore original dimensions of ta - vardef.dimensions = original_dimensions - -def test_only_plev(): +def test_only_plev(monkeypatch): """Test fix.""" + fix = get_allvars_fix('Amon', 'ta') # We know that ta has dimensions time, plev19, latitude, longitude, but the # EMAC CMORizer is designed to check for the presence of each dimension # individually. To test this, remove all but one dimension of ta to create # an artificial, but realistic test case. - vardef = get_var_info('EMAC', 'Amon', 'ta') - original_dimensions = vardef.dimensions - vardef.dimensions = ['plev19'] - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'ta', ()) - fix = AllVars(vardef, extra_facets=extra_facets) + monkeypatch.setattr(fix.vardef, 'dimensions', ['plev19']) # Create cube with only a single dimension - plev_coord = DimCoord([1000.0, 900.0], var_name='plev', - standard_name='air_pressure', units='hPa') + plev_coord = DimCoord([1000.0, 900.0], + var_name='plev', + standard_name='air_pressure', + units='hPa') cubes = CubeList([ - Cube([1, 1], var_name='tm1_p19_ave', units='K', + Cube([1, 1], + var_name='tm1_p19_ave', + units='K', dim_coords_and_dims=[(plev_coord, 0)]), ]) fixed_cubes = fix.fix_metadata(cubes) @@ -621,27 +648,25 @@ def test_only_plev(): np.testing.assert_allclose(new_plev_coord.points, [100000.0, 90000.0]) assert new_plev_coord.bounds is None - # Restore original dimensions of ta - vardef.dimensions = original_dimensions - -def test_only_latitude(): +def test_only_latitude(monkeypatch): """Test fix.""" + fix = get_allvars_fix('Amon', 'ta') # We know that ta has dimensions time, plev19, latitude, longitude, but the # EMAC CMORizer is designed to check for the presence of each dimension # individually. To test this, remove all but one dimension of ta to create # an artificial, but realistic test case. - vardef = get_var_info('EMAC', 'Amon', 'ta') - original_dimensions = vardef.dimensions - vardef.dimensions = ['latitude'] - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'ta', ()) - fix = AllVars(vardef, extra_facets=extra_facets) + monkeypatch.setattr(fix.vardef, 'dimensions', ['latitude']) # Create cube with only a single dimension - lat_coord = DimCoord([0.0, 10.0], var_name='lat', standard_name='latitude', + lat_coord = DimCoord([0.0, 10.0], + var_name='lat', + standard_name='latitude', units='degrees') cubes = CubeList([ - Cube([1, 1], var_name='tm1_p19_ave', units='K', + Cube([1, 1], + var_name='tm1_p19_ave', + units='K', dim_coords_and_dims=[(lat_coord, 0)]), ]) fixed_cubes = fix.fix_metadata(cubes) @@ -666,27 +691,25 @@ def test_only_latitude(): np.testing.assert_allclose(new_lat_coord.bounds, [[-5.0, 5.0], [5.0, 15.0]]) - # Restore original dimensions of ta - vardef.dimensions = original_dimensions - -def test_only_longitude(): +def test_only_longitude(monkeypatch): """Test fix.""" + fix = get_allvars_fix('Amon', 'ta') # We know that ta has dimensions time, plev19, latitude, longitude, but the # EMAC CMORizer is designed to check for the presence of each dimension # individually. To test this, remove all but one dimension of ta to create # an artificial, but realistic test case. - vardef = get_var_info('EMAC', 'Amon', 'ta') - original_dimensions = vardef.dimensions - vardef.dimensions = ['longitude'] - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'ta', ()) - fix = AllVars(vardef, extra_facets=extra_facets) + monkeypatch.setattr(fix.vardef, 'dimensions', ['longitude']) # Create cube with only a single dimension - lon_coord = DimCoord([0.0, 180.0], var_name='lon', - standard_name='longitude', units='degrees') + lon_coord = DimCoord([0.0, 180.0], + var_name='lon', + standard_name='longitude', + units='degrees') cubes = CubeList([ - Cube([1, 1], var_name='tm1_p19_ave', units='K', + Cube([1, 1], + var_name='tm1_p19_ave', + units='K', dim_coords_and_dims=[(lon_coord, 0)]), ]) fixed_cubes = fix.fix_metadata(cubes) @@ -711,9 +734,6 @@ def test_only_longitude(): np.testing.assert_allclose(new_lon_coord.bounds, [[-90.0, 90.0], [90.0, 270.0]]) - # Restore original dimensions of ta - vardef.dimensions = original_dimensions - # Tests with sample data # Note: test_data_path is defined in tests/integration/cmor/_fixes/conftest.py @@ -744,15 +764,13 @@ def test_sample_data_tas(test_data_path, tmp_path): ) -def test_sample_data_ta_plev(test_data_path, tmp_path): +def test_sample_data_ta_plev(test_data_path, tmp_path, monkeypatch): """Test fix.""" + fix = get_allvars_fix('Amon', 'ta') # Note: raw_name needs to be modified since the sample file only contains # plev39, while Amon's ta needs plev19 by default - vardef = get_var_info('EMAC', 'Amon', 'ta') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'ta', ()) - original_raw_name = extra_facets['raw_name'] - extra_facets['raw_name'] = ['tm1_p39_cav', 'tm1_p39_ave'] - fix = AllVars(vardef, extra_facets=extra_facets) + monkeypatch.setitem(fix.extra_facets, 'raw_name', + ['tm1_p39_cav', 'tm1_p39_ave']) filepath = test_data_path / 'emac.nc' fixed_path = fix.fix_file(filepath, tmp_path) @@ -775,8 +793,6 @@ def test_sample_data_ta_plev(test_data_path, tmp_path): rtol=1e-5, ) - fix.extra_facets['raw_name'] = original_raw_name - def test_sample_data_ta_alevel(test_data_path, tmp_path): """Test fix.""" @@ -870,13 +886,7 @@ def test_get_clt_fix(): def test_clt_fix(cubes_2d): """Test fix.""" cubes_2d[0].var_name = 'aclcov_cav' - vardef = get_var_info('EMAC', 'Amon', 'clt') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'clt', ()) - fix = Clt(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_2d) - - fix = get_allvars_fix('Amon', 'clt') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_2d, 'Amon', 'clt') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -901,13 +911,8 @@ def test_clwvi_fix(cubes_2d): cubes_2d[1].var_name = 'xivi_cav' cubes_2d[0].units = 'kg m-2' cubes_2d[1].units = 'kg m-2' - vardef = get_var_info('EMAC', 'Amon', 'clwvi') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'clwvi', ()) - fix = Clwvi(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_2d) - fix = get_allvars_fix('Amon', 'clwvi') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_2d, 'Amon', 'clwvi') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -961,9 +966,7 @@ def test_evspsbl_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - vardef = get_var_info('EMAC', 'Amon', 'evspsbl') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'evspsbl', ()) - fix = Evspsbl(vardef, extra_facets=extra_facets) + fix = get_fix('Amon', 'evspsbl') cube = fix.fix_data(cube) assert cube.var_name == 'evspsbl' @@ -992,9 +995,7 @@ def test_hfls_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - vardef = get_var_info('EMAC', 'Amon', 'hfls') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'hfls', ()) - fix = Hfls(vardef, extra_facets=extra_facets) + fix = get_fix('Amon', 'hfls') cube = fix.fix_data(cube) assert cube.var_name == 'hfls' @@ -1022,9 +1023,7 @@ def test_hfss_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - vardef = get_var_info('EMAC', 'Amon', 'hfss') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'hfss', ()) - fix = Hfss(vardef, extra_facets=extra_facets) + fix = get_fix('Amon', 'hfss') cube = fix.fix_data(cube) assert cube.var_name == 'hfss' @@ -1045,13 +1044,7 @@ def test_get_hurs_fix(): def test_hurs_fix(cubes_2d): """Test fix.""" cubes_2d[0].var_name = 'rh_2m_cav' - vardef = get_var_info('EMAC', 'Amon', 'hurs') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'hurs', ()) - fix = Hurs(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_2d) - - fix = get_allvars_fix('Amon', 'hurs') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_2d, 'Amon', 'hurs') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -1073,13 +1066,7 @@ def test_get_od550aer_fix(): def test_od550aer_fix(cubes_3d): """Test fix.""" cubes_3d[0].var_name = 'aot_opt_TOT_550_total_cav' - vardef = get_var_info('EMAC', 'Amon', 'od550aer') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'od550aer', ()) - fix = Od550aer(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_3d) - - allvars_fix = get_allvars_fix('Amon', 'od550aer') - fixed_cubes = allvars_fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_3d, 'Amon', 'od550aer') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -1107,13 +1094,8 @@ def test_pr_fix(cubes_2d): cubes_2d[1].var_name = 'aprc_cav' cubes_2d[0].units = 'kg m-2 s-1' cubes_2d[1].units = 'kg m-2 s-1' - vardef = get_var_info('EMAC', 'Amon', 'pr') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'pr', ()) - fix = Pr(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_2d) - fix = get_allvars_fix('Amon', 'pr') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_2d, 'Amon', 'pr') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -1282,13 +1264,7 @@ def test_rlds_fix(cubes_2d): cubes_2d[1].var_name = 'tradsu_cav' cubes_2d[0].units = 'W m-2' cubes_2d[1].units = 'W m-2' - vardef = get_var_info('EMAC', 'Amon', 'rlds') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'rlds', ()) - fix = Rlds(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_2d) - - fix = get_allvars_fix('Amon', 'rlds') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_2d, 'Amon', 'rlds') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -1317,9 +1293,7 @@ def test_rlus_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - vardef = get_var_info('EMAC', 'Amon', 'rlus') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'rlus', ()) - fix = Rlus(vardef, extra_facets=extra_facets) + fix = get_fix('Amon', 'rlus') cube = fix.fix_data(cube) assert cube.var_name == 'rlus' @@ -1347,9 +1321,7 @@ def test_rlut_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - vardef = get_var_info('EMAC', 'Amon', 'rlut') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'rlut', ()) - fix = Rlut(vardef, extra_facets=extra_facets) + fix = get_fix('Amon', 'rlut') cube = fix.fix_data(cube) assert cube.var_name == 'rlut' @@ -1377,9 +1349,7 @@ def test_rlutcs_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - vardef = get_var_info('EMAC', 'Amon', 'rlutcs') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'rlutcs', ()) - fix = Rlutcs(vardef, extra_facets=extra_facets) + fix = get_fix('Amon', 'rlutcs') cube = fix.fix_data(cube) assert cube.var_name == 'rlutcs' @@ -1404,13 +1374,7 @@ def test_rsds_fix(cubes_2d): cubes_2d[1].var_name = 'sradsu_cav' cubes_2d[0].units = 'W m-2' cubes_2d[1].units = 'W m-2' - vardef = get_var_info('EMAC', 'Amon', 'rsds') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'rsds', ()) - fix = Rsds(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_2d) - - fix = get_allvars_fix('Amon', 'rsds') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_2d, 'Amon', 'rsds') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -1435,13 +1399,7 @@ def test_rsdt_fix(cubes_2d): cubes_2d[1].var_name = 'srad0u_cav' cubes_2d[0].units = 'W m-2' cubes_2d[1].units = 'W m-2' - vardef = get_var_info('EMAC', 'Amon', 'rsdt') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'rsdt', ()) - fix = Rsdt(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_2d) - - fix = get_allvars_fix('Amon', 'rsdt') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_2d, 'Amon', 'rsdt') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -1470,9 +1428,7 @@ def test_rsus_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - vardef = get_var_info('EMAC', 'Amon', 'rsus') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'rsus', ()) - fix = Rsus(vardef, extra_facets=extra_facets) + fix = get_fix('Amon', 'rsus') cube = fix.fix_data(cube) assert cube.var_name == 'rsus' @@ -1500,9 +1456,7 @@ def test_rsut_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - vardef = get_var_info('EMAC', 'Amon', 'rsut') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'rsut', ()) - fix = Rsut(vardef, extra_facets=extra_facets) + fix = get_fix('Amon', 'rsut') cube = fix.fix_data(cube) assert cube.var_name == 'rsut' @@ -1530,9 +1484,7 @@ def test_rsutcs_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - vardef = get_var_info('EMAC', 'Amon', 'rsutcs') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'rsutcs', ()) - fix = Rsutcs(vardef, extra_facets=extra_facets) + fix = get_fix('Amon', 'rsutcs') cube = fix.fix_data(cube) assert cube.var_name == 'rsutcs' @@ -1557,13 +1509,7 @@ def test_rtmt_fix(cubes_2d): cubes_2d[1].var_name = 'flxstop_cav' cubes_2d[0].units = 'W m-2' cubes_2d[1].units = 'W m-2' - vardef = get_var_info('EMAC', 'Amon', 'rtmt') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'rtmt', ()) - fix = Rtmt(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_2d) - - fix = get_allvars_fix('Amon', 'rtmt') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_2d, 'Amon', 'rtmt') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -1612,13 +1558,7 @@ def test_get_siconc_fix(): def test_siconc_fix(cubes_2d): """Test fix.""" cubes_2d[0].var_name = 'seaice_cav' - vardef = get_var_info('EMAC', 'SImon', 'siconc') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'SImon', 'siconc', ()) - fix = Siconc(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_2d) - - fix = get_allvars_fix('SImon', 'siconc') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_2d, 'SImon', 'siconc') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -1642,9 +1582,7 @@ def test_get_siconca_fix(): def test_siconca_fix(cubes_2d): """Test fix.""" cubes_2d[0].var_name = 'seaice_cav' - vardef = get_var_info('EMAC', 'SImon', 'siconca') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'SImon', 'siconca', ()) - fix = Siconca(vardef, extra_facets=extra_facets) + fix = get_fix('SImon', 'siconca') fixed_cubes = fix.fix_metadata(cubes_2d) fix = get_allvars_fix('SImon', 'siconca') @@ -1679,9 +1617,7 @@ def test_sithick_fix(cubes_2d): assert len(fixed_cubes) == 1 cube = fixed_cubes[0] - vardef = get_var_info('EMAC', 'SImon', 'sithick') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'SImon', 'sithick', ()) - fix = Sithick(vardef, extra_facets=extra_facets) + fix = get_fix('SImon', 'sithick') cube = fix.fix_data(cube) assert cube.var_name == 'sithick' @@ -1860,13 +1796,7 @@ def test_toz_fix(cubes_2d): """Test fix.""" cubes_2d[0].var_name = 'toz' cubes_2d[0].units = 'DU' - vardef = get_var_info('EMAC', 'AERmon', 'toz') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'AERmon', 'toz', ()) - fix = Toz(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_2d) - - fix = get_allvars_fix('AERmon', 'toz') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_2d, 'AERmon', 'toz') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -1975,14 +1905,7 @@ def test_MP_BC_tot_fix(cubes_1d): # noqa: N802 cubes_1d[1].units = 'kg' cubes_1d[2].units = 'kg' cubes_1d[3].units = 'kg' - vardef = get_var_info('EMAC', 'TRAC10hr', 'MP_BC_tot') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'TRAC10hr', 'MP_BC_tot', - ()) - fix = MP_BC_tot(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_1d) - - fix = get_allvars_fix('TRAC10hr', 'MP_BC_tot') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_1d, 'TRAC10hr', 'MP_BC_tot') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -2132,14 +2055,8 @@ def test_MP_DU_tot_fix(cubes_1d): # noqa: N802 cubes_1d[1].units = 'kg' cubes_1d[2].units = 'kg' cubes_1d[3].units = 'kg' - vardef = get_var_info('EMAC', 'TRAC10hr', 'MP_DU_tot') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'TRAC10hr', 'MP_DU_tot', - ()) - fix = MP_DU_tot(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_1d) - fix = get_allvars_fix('TRAC10hr', 'MP_DU_tot') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_1d, 'TRAC10hr', 'MP_DU_tot') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -2385,14 +2302,8 @@ def test_MP_SO4mm_tot_fix(cubes_1d): # noqa: N802 cubes_1d[1].units = 'kg' cubes_1d[2].units = 'kg' cubes_1d[3].units = 'kg' - vardef = get_var_info('EMAC', 'TRAC10hr', 'MP_SO4mm_tot') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'TRAC10hr', 'MP_SO4mm_tot', - ()) - fix = MP_SO4mm_tot(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_1d) - fix = get_allvars_fix('TRAC10hr', 'MP_SO4mm_tot') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_1d, 'TRAC10hr', 'MP_SO4mm_tot') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -2420,14 +2331,8 @@ def test_MP_SS_tot_fix(cubes_1d): # noqa: N802 cubes_1d[0].units = 'kg' cubes_1d[1].units = 'kg' cubes_1d[2].units = 'kg' - vardef = get_var_info('EMAC', 'TRAC10hr', 'MP_SS_tot') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'TRAC10hr', 'MP_SS_tot', - ()) - fix = MP_SS_tot(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_1d) - fix = get_allvars_fix('TRAC10hr', 'MP_SS_tot') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_1d, 'TRAC10hr', 'MP_SS_tot') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -2453,13 +2358,8 @@ def test_get_cl_fix(): def test_cl_fix(cubes_3d): """Test fix.""" cubes_3d[0].var_name = 'aclcac_cav' - vardef = get_var_info('EMAC', 'Amon', 'cl') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'cl', ()) - fix = Cl(vardef, extra_facets=extra_facets) - fixed_cubes = fix.fix_metadata(cubes_3d) - fix = get_allvars_fix('Amon', 'cl') - fixed_cubes = fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_3d, 'Amon', 'cl') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -2671,9 +2571,7 @@ def test_zg_fix(cubes_3d): """Test fix.""" cubes_3d[0].var_name = 'geopot_p19_cav' cubes_3d[0].units = 'm2 s-2' - vardef = get_var_info('EMAC', 'Amon', 'zg') - extra_facets = get_extra_facets('EMAC', 'EMAC', 'Amon', 'zg', ()) - fix = Zg(vardef, extra_facets=extra_facets) + fix = get_fix('Amon', 'zg') fixed_cubes = fix.fix_metadata(cubes_3d) fix = get_allvars_fix('Amon', 'zg') diff --git a/tests/integration/cmor/_fixes/icon/test_icon.py b/tests/integration/cmor/_fixes/icon/test_icon.py index 5b32cbdacf..32dcefa61d 100644 --- a/tests/integration/cmor/_fixes/icon/test_icon.py +++ b/tests/integration/cmor/_fixes/icon/test_icon.py @@ -9,11 +9,13 @@ from iris.coords import AuxCoord, DimCoord from iris.cube import Cube, CubeList +import esmvalcore.cmor._fixes.icon.icon from esmvalcore.cmor._fixes.icon._base_fixes import IconFix from esmvalcore.cmor._fixes.icon.icon import AllVars, Clwvi, Siconc, Siconca from esmvalcore.cmor.fix import Fix from esmvalcore.cmor.table import get_var_info from esmvalcore.config._config import get_extra_facets +from esmvalcore.dataset import Dataset TEST_GRID_FILE_URI = ( 'https://github.com/ESMValGroup/ESMValCore/raw/main/tests/integration/' @@ -87,14 +89,41 @@ def cubes_2d_lat_lon_grid(): return CubeList([cube]) -def get_allvars_fix(mip, short_name): - """Get member of fix class.""" - vardef = get_var_info('ICON', mip, short_name) - extra_facets = get_extra_facets('ICON', 'ICON', mip, short_name, ()) - fix = AllVars(vardef, extra_facets=extra_facets) +def _get_fix(mip, short_name, fix_name): + """Load a fix from esmvalcore.cmor._fixes.icon.icon.""" + dataset = Dataset( + project='ICON', + dataset='ICON', + mip=mip, + short_name=short_name, + ) + extra_facets = get_extra_facets(dataset, ()) + vardef = get_var_info(project='ICON', mip=mip, short_name=short_name) + cls = getattr(esmvalcore.cmor._fixes.icon.icon, fix_name) + fix = cls(vardef, extra_facets=extra_facets) return fix +def get_fix(mip, short_name): + """Load a variable fix from esmvalcore.cmor._fixes.icon.icon.""" + fix_name = short_name[0].upper() + short_name[1:] + return _get_fix(mip, short_name, fix_name) + + +def get_allvars_fix(mip, short_name): + """Load the AllVars fix from esmvalcore.cmor._fixes.icon.icon.""" + return _get_fix(mip, short_name, 'AllVars') + + +def fix_metadata(cubes, mip, short_name): + """Fix metadata of cubes.""" + fix = get_fix(mip, short_name) + cubes = fix.fix_metadata(cubes) + fix = get_allvars_fix(mip, short_name) + cubes = fix.fix_metadata(cubes) + return cubes + + def check_ta_metadata(cubes): """Check ta metadata.""" assert len(cubes) == 1 @@ -489,13 +518,7 @@ def test_clwvi_fix(cubes_regular_grid): cubes[0].units = '1e3 kg m-2' cubes[1].units = '1e3 kg m-2' - vardef = get_var_info('ICON', 'Amon', 'clwvi') - extra_facets = get_extra_facets('ICON', 'ICON', 'Amon', 'clwvi', ()) - clwvi_fix = Clwvi(vardef, extra_facets=extra_facets) - allvars_fix = get_allvars_fix('Amon', 'clwvi') - - fixed_cubes = clwvi_fix.fix_metadata(cubes) - fixed_cubes = allvars_fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes, 'Amon', 'clwvi') assert len(fixed_cubes) == 1 cube = fixed_cubes[0] @@ -596,13 +619,7 @@ def test_get_siconc_fix(): def test_siconc_fix(cubes_2d): """Test fix.""" - vardef = get_var_info('ICON', 'SImon', 'siconc') - extra_facets = get_extra_facets('ICON', 'ICON', 'SImon', 'siconc', ()) - siconc_fix = Siconc(vardef, extra_facets=extra_facets) - allvars_fix = get_allvars_fix('SImon', 'siconc') - - fixed_cubes = siconc_fix.fix_metadata(cubes_2d) - fixed_cubes = allvars_fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_2d, 'SImon', 'siconc') cube = check_siconc_metadata(fixed_cubes, 'siconc', 'Sea-Ice Area Percentage (Ocean Grid)') @@ -624,13 +641,7 @@ def test_get_siconca_fix(): def test_siconca_fix(cubes_2d): """Test fix.""" - vardef = get_var_info('ICON', 'SImon', 'siconca') - extra_facets = get_extra_facets('ICON', 'ICON', 'SImon', 'siconca', ()) - siconca_fix = Siconca(vardef, extra_facets=extra_facets) - allvars_fix = get_allvars_fix('SImon', 'siconca') - - fixed_cubes = siconca_fix.fix_metadata(cubes_2d) - fixed_cubes = allvars_fix.fix_metadata(fixed_cubes) + fixed_cubes = fix_metadata(cubes_2d, 'SImon', 'siconca') cube = check_siconc_metadata(fixed_cubes, 'siconca', 'Sea-Ice Area Percentage (Atmospheric Grid)') @@ -879,17 +890,14 @@ def test_2d_lat_lon_grid_fix(cubes_2d_lat_lon_grid): # Test fix with empty standard_name -def test_empty_standard_name_fix(cubes_2d): +def test_empty_standard_name_fix(cubes_2d, monkeypatch): """Test fix.""" + fix = get_allvars_fix('Amon', 'tas') # We know that tas has a standard name, but this being native model output # there may be variables with no standard name. The code is designed to # handle this gracefully and here we test it with an artificial, but # realistic case. - vardef = get_var_info('ICON', 'Amon', 'tas') - original_standard_name = vardef.standard_name - vardef.standard_name = '' - extra_facets = get_extra_facets('ICON', 'ICON', 'Amon', 'tas', ()) - fix = AllVars(vardef, extra_facets=extra_facets) + monkeypatch.setattr(fix.vardef, 'standard_name', '') fixed_cubes = fix.fix_metadata(cubes_2d) assert len(fixed_cubes) == 1 @@ -900,9 +908,6 @@ def test_empty_standard_name_fix(cubes_2d): assert cube.units == 'K' assert 'positive' not in cube.attributes - # Restore original standard_name of tas - vardef.standard_name = original_standard_name - # Test automatic addition of missing coordinates @@ -1136,18 +1141,19 @@ def test_get_horizontal_grid_cache_file_too_old(tmp_path, monkeypatch): def test_only_time(monkeypatch): """Test fix.""" + fix = get_allvars_fix('Amon', 'ta') # We know that ta has dimensions time, plev19, latitude, longitude, but the # ICON CMORizer is designed to check for the presence of each dimension # individually. To test this, remove all but one dimension of ta to create # an artificial, but realistic test case. - vardef = get_var_info('ICON', 'Amon', 'ta') - monkeypatch.setattr(vardef, 'dimensions', ['time']) - extra_facets = get_extra_facets('ICON', 'ICON', 'Amon', 'ta', ()) - fix = AllVars(vardef, extra_facets=extra_facets) + monkeypatch.setattr(fix.vardef, 'dimensions', ['time']) # Create cube with only a single dimension - time_coord = DimCoord([0.0, 1.0], var_name='time', standard_name='time', - long_name='time', units='days since 1850-01-01') + time_coord = DimCoord([0.0, 1.0], + var_name='time', + standard_name='time', + long_name='time', + units='days since 1850-01-01') cubes = CubeList([ Cube([1, 1], var_name='ta', units='K', dim_coords_and_dims=[(time_coord, 0)]), @@ -1180,18 +1186,18 @@ def test_only_time(monkeypatch): def test_only_height(monkeypatch): """Test fix.""" + fix = get_allvars_fix('Amon', 'ta') # We know that ta has dimensions time, plev19, latitude, longitude, but the # ICON CMORizer is designed to check for the presence of each dimension # individually. To test this, remove all but one dimension of ta to create # an artificial, but realistic test case. - vardef = get_var_info('ICON', 'Amon', 'ta') - monkeypatch.setattr(vardef, 'dimensions', ['plev19']) - extra_facets = get_extra_facets('ICON', 'ICON', 'Amon', 'ta', ()) - fix = AllVars(vardef, extra_facets=extra_facets) + monkeypatch.setattr(fix.vardef, 'dimensions', ['plev19']) # Create cube with only a single dimension - height_coord = DimCoord([1000.0, 100.0], var_name='height', - standard_name='height', units='cm') + height_coord = DimCoord([1000.0, 100.0], + var_name='height', + standard_name='height', + units='cm') cubes = CubeList([ Cube([1, 1], var_name='ta', units='K', dim_coords_and_dims=[(height_coord, 0)]), @@ -1224,17 +1230,17 @@ def test_only_height(monkeypatch): def test_only_latitude(monkeypatch): """Test fix.""" + fix = get_allvars_fix('Amon', 'ta') # We know that ta has dimensions time, plev19, latitude, longitude, but the # ICON CMORizer is designed to check for the presence of each dimension # individually. To test this, remove all but one dimension of ta to create # an artificial, but realistic test case. - vardef = get_var_info('ICON', 'Amon', 'ta') - monkeypatch.setattr(vardef, 'dimensions', ['latitude']) - extra_facets = get_extra_facets('ICON', 'ICON', 'Amon', 'ta', ()) - fix = AllVars(vardef, extra_facets=extra_facets) + monkeypatch.setattr(fix.vardef, 'dimensions', ['latitude']) # Create cube with only a single dimension - lat_coord = DimCoord([0.0, 10.0], var_name='lat', standard_name='latitude', + lat_coord = DimCoord([0.0, 10.0], + var_name='lat', + standard_name='latitude', units='degrees') cubes = CubeList([ Cube([1, 1], var_name='ta', units='K', @@ -1267,18 +1273,18 @@ def test_only_latitude(monkeypatch): def test_only_longitude(monkeypatch): """Test fix.""" + fix = get_allvars_fix('Amon', 'ta') # We know that ta has dimensions time, plev19, latitude, longitude, but the # ICON CMORizer is designed to check for the presence of each dimension # individually. To test this, remove all but one dimension of ta to create # an artificial, but realistic test case. - vardef = get_var_info('ICON', 'Amon', 'ta') - monkeypatch.setattr(vardef, 'dimensions', ['longitude']) - extra_facets = get_extra_facets('ICON', 'ICON', 'Amon', 'ta', ()) - fix = AllVars(vardef, extra_facets=extra_facets) + monkeypatch.setattr(fix.vardef, 'dimensions', ['longitude']) # Create cube with only a single dimension - lon_coord = DimCoord([0.0, 180.0], var_name='lon', - standard_name='longitude', units='degrees') + lon_coord = DimCoord([0.0, 180.0], + var_name='lon', + standard_name='longitude', + units='degrees') cubes = CubeList([ Cube([1, 1], var_name='ta', units='K', dim_coords_and_dims=[(lon_coord, 0)]), diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index d40e0dbea0..1dbb948940 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -21,6 +21,9 @@ def session(tmp_path, monkeypatch): monkeypatch.setitem(CFG, 'drs', {}) for project in _config.CFG: monkeypatch.setitem(_config.CFG[project]['input_dir'], 'default', '/') + # The patched datafinder fixture does not return any facets, so automatic + # supplementary definition does not work with it. + session['use_legacy_supplementaries'] = True return session @@ -64,7 +67,6 @@ def _get_filenames(root_path, filename, tracking_id): @pytest.fixture def patched_datafinder(tmp_path, monkeypatch): - def tracking_ids(i=0): while True: yield i @@ -80,7 +82,6 @@ def glob(file_glob): @pytest.fixture def patched_failing_datafinder(tmp_path, monkeypatch): - def tracking_ids(i=0): while True: yield i diff --git a/tests/integration/dataset/test_dataset.py b/tests/integration/dataset/test_dataset.py index 9c6e8142a9..0c94dc8c48 100644 --- a/tests/integration/dataset/test_dataset.py +++ b/tests/integration/dataset/test_dataset.py @@ -44,7 +44,7 @@ def test_load(example_data): exp='historical', timerange='1850/185002', ) - tas.add_ancillary(short_name='areacella', mip='fx', ensemble='r0i0p0') + tas.add_supplementary(short_name='areacella', mip='fx', ensemble='r0i0p0') tas.augment_facets() diff --git a/tests/integration/esgf/search_results/expected.yml b/tests/integration/esgf/search_results/expected.yml index b5b5da7784..9e463c0cf7 100644 --- a/tests/integration/esgf/search_results/expected.yml +++ b/tests/integration/esgf/search_results/expected.yml @@ -18,7 +18,7 @@ Amon_r1i1p1_historical,rcp85_INM-CM4_CMIP5_tas.json: mip: Amon product: output1 project: CMIP5 - realm: atmos + modeling_realm: atmos short_name: tas version: v20130207 local_file: cmip5/output1/INM/inmcm4/historical/mon/atmos/Amon/r1i1p1/v20130207/tas_Amon_inmcm4_historical_r1i1p1_185001-200512.nc @@ -48,7 +48,7 @@ Amon_r1i1p1_historical,rcp85_INM-CM4_CMIP5_tas.json: mip: Amon product: output1 project: CMIP5 - realm: atmos + modeling_realm: atmos short_name: tas version: v20130207 local_file: cmip5/output1/INM/inmcm4/rcp85/mon/atmos/Amon/r1i1p1/v20130207/tas_Amon_inmcm4_rcp85_r1i1p1_200601-210012.nc @@ -79,7 +79,7 @@ Amon_r1i1p1_historical_FIO-ESM_CMIP5_tas.json: mip: Amon product: output1 project: CMIP5 - realm: atmos + modeling_realm: atmos short_name: tas version: v20121010 local_file: cmip5/output1/FIO/FIO-ESM/historical/mon/atmos/Amon/r1i1p1/v20121010/tas_Amon_FIO-ESM_historical_r1i1p1_185001-200512.nc @@ -106,7 +106,7 @@ Amon_r1i1p1_rcp85_HadGEM2-CC_CMIP5_tas.json: mip: Amon product: output1 project: CMIP5 - realm: atmos + modeling_realm: atmos short_name: tas version: v20120531 local_file: cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_205512-208011.nc @@ -130,7 +130,7 @@ Amon_r1i1p1_rcp85_HadGEM2-CC_CMIP5_tas.json: mip: Amon product: output1 project: CMIP5 - realm: atmos + modeling_realm: atmos short_name: tas version: v20120531 local_file: cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_208012-209912.nc @@ -154,7 +154,7 @@ Amon_r1i1p1_rcp85_HadGEM2-CC_CMIP5_tas.json: mip: Amon product: output1 project: CMIP5 - realm: atmos + modeling_realm: atmos short_name: tas version: v20120531 local_file: cmip5/output1/MOHC/HadGEM2-CC/rcp85/mon/atmos/Amon/r1i1p1/v20120531/tas_Amon_HadGEM2-CC_rcp85_r1i1p1_210001-210012.nc @@ -254,7 +254,7 @@ obs4MIPs_CERES-EBAF_mon_rsutcs.json: frequency: mon institute: NASA-LaRC project: obs4MIPs - realm: atmos + modeling_realm: atmos short_name: rsutcs version: v20160610 local_file: obs4MIPs/CERES-EBAF/v20160610/rsutcs_CERES-EBAF_L3B_Ed2-8_200003-201404.nc @@ -272,7 +272,6 @@ obs4MIPs_GPCP-V2.3_pr.json: frequency: mon institute: NASA-GSFC project: obs4MIPs - realm: atmos short_name: pr version: v20180519 local_file: obs4MIPs/GPCP-V2.3/v20180519/pr_GPCP-SG_L3_v2.3_197901-201710.nc @@ -292,7 +291,7 @@ run1_historical_cccma_cgcm3_1_CMIP3_mon_tas.json: frequency: mon institute: CCCma project: CMIP3 - realm: atmos + modeling_realm: atmos short_name: tas version: v1 local_file: cmip3/CCCma/cccma_cgcm3_1/historical/mon/atmos/run1/tas/v1/tas_a1_20c3m_1_cgcm3.1_t47_1850_2000.nc diff --git a/tests/integration/preprocessor/_ancillary_vars/__init__.py b/tests/integration/preprocessor/_ancillary_vars/__init__.py deleted file mode 100644 index 88b606fc48..0000000000 --- a/tests/integration/preprocessor/_ancillary_vars/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -Test _ancillary_vars.py - -Integration tests for the esmvalcore.preprocessor._ancillary_vars module -""" diff --git a/tests/integration/preprocessor/_io/test_load.py b/tests/integration/preprocessor/_io/test_load.py index cc68b88cb0..99247bdc5a 100644 --- a/tests/integration/preprocessor/_io/test_load.py +++ b/tests/integration/preprocessor/_io/test_load.py @@ -21,6 +21,7 @@ def _create_sample_cube(): class TestLoad(unittest.TestCase): """Tests for :func:`esmvalcore.preprocessor.load`.""" + def setUp(self): """Start tests.""" self.temp_files = [] @@ -59,7 +60,7 @@ def test_callback_remove_attributes(self): cube.attributes[attr] = attr self._save_cube(cube) for temp_file in self.temp_files: - cubes = load(temp_file, callback=concatenate_callback) + cubes = load(temp_file, callback='default') cube = cubes[0] self.assertEqual(1, len(cubes)) self.assertTrue((cube.data == np.array([1, 2])).all()) @@ -78,7 +79,7 @@ def test_callback_remove_attributes_from_coords(self): coord.attributes[attr] = attr self._save_cube(cube) for temp_file in self.temp_files: - cubes = load(temp_file, callback=concatenate_callback) + cubes = load(temp_file, callback='default') cube = cubes[0] self.assertEqual(1, len(cubes)) self.assertTrue((cube.data == np.array([1, 2])).all()) diff --git a/tests/integration/preprocessor/_mask/test_mask.py b/tests/integration/preprocessor/_mask/test_mask.py index 918d131fdd..298dc3fb22 100644 --- a/tests/integration/preprocessor/_mask/test_mask.py +++ b/tests/integration/preprocessor/_mask/test_mask.py @@ -1,21 +1,17 @@ -""" -Test mask. - -Integration tests for the :func:`esmvalcore.preprocessor._mask` -module. +"""Test mask. +Integration tests for the :func:`esmvalcore.preprocessor._mask` module. """ -from unittest import mock +from pathlib import Path import iris import iris.fileformats import numpy as np import pytest -from esmvalcore.cmor.check import CheckLevels from esmvalcore.preprocessor import ( PreprocessorFile, - add_fx_variables, + add_supplementary_variables, mask_fillvalues, mask_landsea, mask_landseaice, @@ -68,27 +64,18 @@ def setUp(self): self.mock_data = np.ma.empty((4, 3, 3)) self.mock_data[:] = 10. - def test_components_fx_var(self, tmp_path): + def test_components_fx_var(self): """Test compatibility of ancillary variables.""" self.fx_mask.var_name = 'sftlf' self.fx_mask.standard_name = 'land_area_fraction' - sftlf_file = str(tmp_path / 'sftlf_mask.nc') - iris.save(self.fx_mask, sftlf_file) - fx_vars = { - 'sftlf': { - 'short_name': 'sftlf', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'fx', - 'frequency': 'fx', - 'filename': sftlf_file} - } new_cube_land = iris.cube.Cube( self.new_cube_data, dim_coords_and_dims=self.cube_coords_spec ) - new_cube_land = add_fx_variables(new_cube_land, fx_vars, - CheckLevels.IGNORE) + new_cube_land = add_supplementary_variables( + new_cube_land, + [self.fx_mask], + ) result_land = mask_landsea( new_cube_land, 'land', @@ -97,56 +84,40 @@ def test_components_fx_var(self, tmp_path): self.fx_mask.var_name = 'sftgif' self.fx_mask.standard_name = 'land_ice_area_fraction' - sftgif_file = str(tmp_path / 'sftgif_mask.nc') - iris.save(self.fx_mask, sftgif_file) - fx_vars = { - 'sftgif': { - 'short_name': 'sftgif', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'fx', - 'frequency': 'fx', - 'filename': sftlf_file} - } new_cube_ice = iris.cube.Cube( self.new_cube_data, dim_coords_and_dims=self.cube_coords_spec ) - new_cube_ice = add_fx_variables(new_cube_ice, fx_vars, - CheckLevels.IGNORE) + new_cube_ice = add_supplementary_variables( + new_cube_ice, + [self.fx_mask], + ) result_ice = mask_landseaice( new_cube_ice, 'ice', ) assert isinstance(result_ice, iris.cube.Cube) - def test_mask_landsea(self, tmp_path): + def test_mask_landsea(self): """Test mask_landsea func.""" self.fx_mask.var_name = 'sftlf' self.fx_mask.standard_name = 'land_area_fraction' - sftlf_file = str(tmp_path / 'sftlf_mask.nc') - iris.save(self.fx_mask, sftlf_file) - fx_vars = { - 'sftlf': { - 'short_name': 'sftlf', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'fx', - 'frequency': 'fx', - 'filename': sftlf_file} - } new_cube_land = iris.cube.Cube( self.new_cube_data, dim_coords_and_dims=self.cube_coords_spec ) - new_cube_land = add_fx_variables(new_cube_land, fx_vars, - CheckLevels.IGNORE) + new_cube_land = add_supplementary_variables( + new_cube_land, + [self.fx_mask], + ) new_cube_sea = iris.cube.Cube( self.new_cube_data, dim_coords_and_dims=self.cube_coords_spec ) - new_cube_sea = add_fx_variables(new_cube_sea, fx_vars, - CheckLevels.IGNORE) + new_cube_sea = add_supplementary_variables( + new_cube_sea, + [self.fx_mask], + ) # mask with fx files result_land = mask_landsea( @@ -190,27 +161,18 @@ def test_mask_landsea(self, tmp_path): expected.mask = np.ones((3, 3), bool) assert_array_equal(result_sea.data, expected) - def test_mask_landseaice(self, tmp_path): + def test_mask_landseaice(self): """Test mask_landseaice func.""" self.fx_mask.var_name = 'sftgif' self.fx_mask.standard_name = 'land_ice_area_fraction' - sftgif_file = str(tmp_path / 'sftgif_mask.nc') - iris.save(self.fx_mask, sftgif_file) - fx_vars = { - 'sftgif': { - 'short_name': 'sftgif', - 'project': 'CMIP6', - 'dataset': 'EC-Earth3', - 'mip': 'fx', - 'frequency': 'fx', - 'filename': sftgif_file} - } new_cube_ice = iris.cube.Cube( self.new_cube_data, dim_coords_and_dims=self.cube_coords_spec ) - new_cube_ice = add_fx_variables(new_cube_ice, fx_vars, - CheckLevels.IGNORE) + new_cube_ice = add_supplementary_variables( + new_cube_ice, + [self.fx_mask], + ) result_ice = mask_landseaice(new_cube_ice, 'ice') expected = np.ma.empty((2, 3, 3)) expected.data[:] = 200. @@ -220,20 +182,28 @@ def test_mask_landseaice(self, tmp_path): np.ma.set_fill_value(expected, 1e+20) assert_array_equal(result_ice.data, expected) - def test_mask_fillvalues(self, tmp_path): + def test_mask_fillvalues(self, mocker): """Test the fillvalues mask: func mask_fillvalues.""" data_1 = data_2 = self.mock_data data_2.mask = np.ones((4, 3, 3), bool) coords_spec = [(self.times, 0), (self.lats, 1), (self.lons, 2)] cube_1 = iris.cube.Cube(data_1, dim_coords_and_dims=coords_spec) cube_2 = iris.cube.Cube(data_2, dim_coords_and_dims=coords_spec) - filename_1 = str(tmp_path / 'file1.nc') - filename_2 = str(tmp_path / 'file2.nc') - product_1 = PreprocessorFile(attributes={'filename': filename_1}, - settings={}) + filename_1 = 'file1.nc' + filename_2 = 'file2.nc' + product_1 = mocker.create_autospec( + PreprocessorFile, + spec_set=True, + instance=True, + ) + product_1.filename = filename_1 product_1.cubes = [cube_1] - product_2 = PreprocessorFile(attributes={'filename': filename_2}, - settings={}) + product_2 = mocker.create_autospec( + PreprocessorFile, + spec_set=True, + instance=True, + ) + product_2.filename = filename_2 product_2.cubes = [cube_2] results = mask_fillvalues({product_1, product_2}, 0.95, @@ -248,7 +218,7 @@ def test_mask_fillvalues(self, tmp_path): assert_array_equal(result_2.data.mask, data_2.mask) assert_array_equal(result_1.data, data_1) - def test_mask_fillvalues_zero_threshold(self, tmp_path): + def test_mask_fillvalues_zero_threshold(self, mocker): """Test the fillvalues mask: func mask_fillvalues for 0-threshold.""" data_1 = self.mock_data data_2 = self.mock_data[0:3] @@ -262,13 +232,21 @@ def test_mask_fillvalues_zero_threshold(self, tmp_path): coords_spec2 = [(self.time2, 0), (self.lats, 1), (self.lons, 2)] cube_1 = iris.cube.Cube(data_1, dim_coords_and_dims=coords_spec) cube_2 = iris.cube.Cube(data_2, dim_coords_and_dims=coords_spec2) - filename_1 = str(tmp_path / 'file1.nc') - filename_2 = str(tmp_path / 'file2.nc') - product_1 = PreprocessorFile(attributes={'filename': filename_1}, - settings={}) + filename_1 = Path('file1.nc') + filename_2 = Path('file2.nc') + product_1 = mocker.create_autospec( + PreprocessorFile, + spec_set=True, + instance=True, + ) + product_1.filename = filename_1 product_1.cubes = [cube_1] - product_2 = PreprocessorFile(attributes={'filename': filename_2}, - settings={}) + product_2 = mocker.create_autospec( + PreprocessorFile, + spec_set=True, + instance=True, + ) + product_2.filename = filename_2 product_2.cubes = [cube_2] results = mask_fillvalues({product_1, product_2}, 0., min_value=-1.e20) result_1, result_2 = None, None @@ -282,12 +260,12 @@ def test_mask_fillvalues_zero_threshold(self, tmp_path): result_2.data[0, ...].mask, result_1.data[0, ...].mask, ) - # identical masks with cumluative + # identical masks with cumulative cumulative_mask = cube_1[1:2].data.mask | cube_2[1:2].data.mask assert_array_equal(result_1[1:2].data.mask, cumulative_mask) assert_array_equal(result_2[2:3].data.mask, cumulative_mask) - def test_mask_fillvalues_min_value_none(self, tmp_path): + def test_mask_fillvalues_min_value_none(self, mocker): """Test ``mask_fillvalues`` for min_value=None.""" # We use non-masked data here and explicitly set some values to 0 here # since this caused problems in the past, see @@ -300,14 +278,22 @@ def test_mask_fillvalues_min_value_none(self, tmp_path): coords_spec2 = [(self.time2, 0), (self.lats, 1), (self.lons, 2)] cube_1 = iris.cube.Cube(data_1, dim_coords_and_dims=coords_spec) cube_2 = iris.cube.Cube(data_2, dim_coords_and_dims=coords_spec2) - filename_1 = str(tmp_path / 'file1.nc') - filename_2 = str(tmp_path / 'file2.nc') + filename_1 = Path('file1.nc') + filename_2 = Path('file2.nc') # Mock PreprocessorFile to avoid provenance errors - product_1 = mock.create_autospec(PreprocessorFile, instance=True) + product_1 = mocker.create_autospec( + PreprocessorFile, + spec_set=True, + instance=True, + ) product_1.filename = filename_1 product_1.cubes = [cube_1] - product_2 = mock.create_autospec(PreprocessorFile, instance=True) + product_2 = mocker.create_autospec( + PreprocessorFile, + spec_set=True, + instance=True, + ) product_2.filename = filename_2 product_2.cubes = [cube_2] diff --git a/tests/integration/preprocessor/_regrid/test_get_file_levels.py b/tests/integration/preprocessor/_regrid/test_get_file_levels.py index 128a074453..9ca33d7145 100644 --- a/tests/integration/preprocessor/_regrid/test_get_file_levels.py +++ b/tests/integration/preprocessor/_regrid/test_get_file_levels.py @@ -1,57 +1,34 @@ -""" -Integration tests for the :func: -`esmvalcore.preprocessor.regrid.get_cmor_levels` -function. - -""" - -import os -import tempfile -import unittest - -import iris +"""Integration test for +:func:`esmvalcore.preprocessor.regrid.get_reference_levels`.""" import iris.coords import iris.cube +import iris.util import numpy as np +import pytest +from esmvalcore.dataset import Dataset from esmvalcore.preprocessor import _regrid -class TestGetFileLevels(unittest.TestCase): - def setUp(self): - """Prepare the sample file for the test""" - self.cube = iris.cube.Cube(np.ones([2, 2, 2]), var_name='var') - self.cube.add_dim_coord( - iris.coords.DimCoord(np.arange(0, 2), var_name='coord'), 0) +@pytest.fixture +def test_cube(): + cube = iris.cube.Cube(np.ones([2, 2, 2]), var_name='var') + coord = iris.coords.DimCoord(np.arange(0, 2), var_name='coord') + coord.attributes['positive'] = 'up' + cube.add_dim_coord(coord, 0) + return cube + - self.cube.coord('coord').attributes['positive'] = 'up' - iris.util.guess_coord_axis(self.cube.coord('coord')) - descriptor, self.path = tempfile.mkstemp('.nc') - os.close(descriptor) - print(self.cube) - iris.save(self.cube, self.path) +def test_get_file_levels_from_coord(mocker, test_cube): + dataset = mocker.create_autospec(Dataset, spec_set=True, instance=True) + dataset.copy.return_value.load.return_value = test_cube + reference_levels = _regrid.get_reference_levels(dataset) + assert reference_levels == [0., 1] - def tearDown(self): - """Remove the sample file for the test""" - os.remove(self.path) - def test_get_coord(self): - fix_file = unittest.mock.create_autospec(_regrid.fix_file) - fix_file.side_effect = lambda file, **_: file - fix_metadata = unittest.mock.create_autospec(_regrid.fix_metadata) - fix_metadata.side_effect = lambda cubes, **_: cubes - with unittest.mock.patch('esmvalcore.preprocessor._regrid.fix_file', - fix_file): - with unittest.mock.patch( - 'esmvalcore.preprocessor._regrid.fix_metadata', - fix_metadata): - reference_levels = _regrid.get_reference_levels( - filename=self.path, - project='CMIP6', - dataset='dataset', - short_name='short_name', - mip='mip', - frequency='mon', - fix_dir='output_dir', - ) - self.assertListEqual(reference_levels, [0., 1]) +def test_get_file_levels_from_coord_fail(mocker, test_cube): + test_cube.coord('coord').attributes.clear() + dataset = mocker.create_autospec(Dataset, spec_set=True, instance=True) + dataset.copy.return_value.load.return_value = test_cube + with pytest.raises(ValueError): + _regrid.get_reference_levels(dataset) diff --git a/tests/integration/preprocessor/_regrid/test_regrid.py b/tests/integration/preprocessor/_regrid/test_regrid.py index 0a4529b32e..de7a20749d 100644 --- a/tests/integration/preprocessor/_regrid/test_regrid.py +++ b/tests/integration/preprocessor/_regrid/test_regrid.py @@ -6,14 +6,18 @@ import iris import numpy as np +import pytest from numpy import ma -import tests +from esmvalcore.dataset import Dataset from esmvalcore.preprocessor import regrid +from tests import assert_array_equal from tests.unit.preprocessor._regrid import _make_cube -class Test(tests.Test): +class Test: + + @pytest.fixture(autouse=True) def setUp(self): """Prepare tests.""" shape = (3, 2, 2) @@ -91,7 +95,28 @@ def setUp(self): def test_regrid__linear(self): result = regrid(self.cube, self.grid_for_linear, 'linear') expected = np.array([[[1.5]], [[5.5]], [[9.5]]]) - self.assert_array_equal(result.data, expected) + assert_array_equal(result.data, expected) + + def test_regrid__linear_file(self, tmp_path): + file = tmp_path / "file.nc" + iris.save(self.grid_for_linear, target=file) + result = regrid(self.cube, file, 'linear') + expected = np.array([[[1.5]], [[5.5]], [[9.5]]]) + assert_array_equal(result.data, expected) + + def test_regrid__linear_dataset(self, monkeypatch): + monkeypatch.setattr(Dataset, 'files', ["file.nc"]) + + def load(_): + return self.grid_for_linear + + monkeypatch.setattr(Dataset, 'load', load) + dataset = Dataset( + short_name='tas', + ) + result = regrid(self.cube, dataset, 'linear') + expected = np.array([[[1.5]], [[5.5]], [[9.5]]]) + assert_array_equal(result.data, expected) def test_regrid__esmf_rectilinear(self): scheme_name = 'esmf_regrid.schemes:regrid_rectilinear_to_rectilinear' @@ -126,7 +151,7 @@ def test_regrid__linear_do_not_preserve_dtype(self): self.cube.data = self.cube.data.astype(int) result = regrid(self.cube, self.grid_for_linear, 'linear') expected = np.array([[[1.5]], [[5.5]], [[9.5]]]) - self.assert_array_equal(result.data, expected) + assert_array_equal(result.data, expected) assert np.issubdtype(self.cube.dtype, np.integer) assert np.issubdtype(result.dtype, np.floating) @@ -148,7 +173,7 @@ def test_regrid__linear_extrapolate(self): expected = [[[-3., -1.5, 0.], [0., 1.5, 3.], [3., 4.5, 6.]], [[1., 2.5, 4.], [4., 5.5, 7.], [7., 8.5, 10.]], [[5., 6.5, 8.], [8., 9.5, 11.], [11., 12.5, 14.]]] - self.assert_array_equal(result.data, expected) + assert_array_equal(result.data, expected) def test_regrid__linear_extrapolate_with_mask(self): data = np.empty((3, 3)) @@ -169,7 +194,7 @@ def test_regrid__linear_extrapolate_with_mask(self): expected = ma.empty((3, 3, 3)) expected.mask = ma.masked expected[:, 1, 1] = np.array([1.5, 5.5, 9.5]) - self.assert_array_equal(result.data, expected) + assert_array_equal(result.data, expected) def test_regrid__nearest(self): data = np.empty((1, 1)) @@ -187,7 +212,7 @@ def test_regrid__nearest(self): grid = iris.cube.Cube(data, dim_coords_and_dims=coords_spec) result = regrid(self.cube, grid, 'nearest') expected = np.array([[[3]], [[7]], [[11]]]) - self.assert_array_equal(result.data, expected) + assert_array_equal(result.data, expected) def test_regrid__nearest_extrapolate_with_mask(self): data = np.empty((3, 3)) @@ -207,7 +232,7 @@ def test_regrid__nearest_extrapolate_with_mask(self): expected = ma.empty((3, 3, 3)) expected.mask = ma.masked expected[:, 1, 1] = np.array([3, 7, 11]) - self.assert_array_equal(result.data, expected) + assert_array_equal(result.data, expected) def test_regrid__area_weighted(self): data = np.empty((1, 1)) diff --git a/tests/integration/preprocessor/_supplementary_vars/__init__.py b/tests/integration/preprocessor/_supplementary_vars/__init__.py new file mode 100644 index 0000000000..1cfa4148e4 --- /dev/null +++ b/tests/integration/preprocessor/_supplementary_vars/__init__.py @@ -0,0 +1,4 @@ +"""Test _supplementary_vars.py. + +Integration tests for `esmvalcore.preprocessor._supplementary_vars`. +""" diff --git a/tests/integration/preprocessor/_ancillary_vars/test_add_fx_variables.py b/tests/integration/preprocessor/_supplementary_vars/test_add_fx_variables.py similarity index 98% rename from tests/integration/preprocessor/_ancillary_vars/test_add_fx_variables.py rename to tests/integration/preprocessor/_supplementary_vars/test_add_fx_variables.py index d887204b40..415a5849c4 100644 --- a/tests/integration/preprocessor/_ancillary_vars/test_add_fx_variables.py +++ b/tests/integration/preprocessor/_supplementary_vars/test_add_fx_variables.py @@ -1,7 +1,7 @@ """Test add_fx_variables. Integration tests for the -:func:`esmvalcore.preprocessor._ancillary_vars` module. +:func:`esmvalcore.preprocessor._supplementary_vars` module. """ import logging @@ -10,7 +10,7 @@ import pytest from esmvalcore.cmor.check import CheckLevels -from esmvalcore.preprocessor._ancillary_vars import ( +from esmvalcore.preprocessor._supplementary_vars import ( _is_fx_broadcastable, add_ancillary_variable, add_cell_measure, @@ -219,7 +219,7 @@ def test_no_cell_measure(self): cube = add_fx_variables(cube, {'areacello': None}, CheckLevels.IGNORE) assert cube.cell_measures() == [] - def test_add_ancillary_vars(self, tmp_path): + def test_add_ancillary_variables(self, tmp_path): """Test invalid variable is not added as cell measure.""" self.fx_area.var_name = 'sftlf' self.fx_area.standard_name = "land_area_fraction" diff --git a/tests/integration/preprocessor/_supplementary_vars/test_add_supplementary_variables.py b/tests/integration/preprocessor/_supplementary_vars/test_add_supplementary_variables.py new file mode 100644 index 0000000000..9a3fe8169c --- /dev/null +++ b/tests/integration/preprocessor/_supplementary_vars/test_add_supplementary_variables.py @@ -0,0 +1,174 @@ +"""Test add_supplementary_variables and remove_supplementary_variables. + +Integration tests for the +:func:`esmvalcore.preprocessor._supplementary_vars` module. +""" +import iris +import iris.fileformats +import numpy as np +import pytest + +import esmvalcore.config +from esmvalcore.preprocessor._supplementary_vars import ( + add_ancillary_variable, + add_cell_measure, + add_supplementary_variables, + remove_supplementary_variables, +) + + +class Test: + """Test class.""" + @pytest.fixture(autouse=True) + def setUp(self): + """Assemble a stock cube.""" + fx_area_data = np.ones((3, 3)) + fx_volume_data = np.ones((3, 3, 3)) + self.new_cube_data = np.empty((3, 3)) + self.new_cube_data[:] = 200. + self.new_cube_3D_data = np.empty((3, 3, 3)) + self.new_cube_3D_data[:] = 200. + crd_sys = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) + self.lons = iris.coords.DimCoord([0, 1.5, 3], + standard_name='longitude', + bounds=[[0, 1], [1, 2], [2, 3]], + units='degrees_east', + coord_system=crd_sys) + self.lats = iris.coords.DimCoord([0, 1.5, 3], + standard_name='latitude', + bounds=[[0, 1], [1, 2], [2, 3]], + units='degrees_north', + coord_system=crd_sys) + self.depth = iris.coords.DimCoord([0, 1.5, 3], + standard_name='depth', + bounds=[[0, 1], [1, 2], [2, 3]], + units='m', + long_name='ocean depth coordinate') + self.monthly_times = iris.coords.DimCoord( + [15.5, 45, 74.5, 105, 135.5, 166, + 196.5, 227.5, 258, 288.5, 319, 349.5], + standard_name='time', + var_name='time', + bounds=[[0, 31], [31, 59], [59, 90], + [90, 120], [120, 151], [151, 181], + [181, 212], [212, 243], [243, 273], + [273, 304], [304, 334], [334, 365]], + units='days since 1950-01-01 00:00:00') + self.yearly_times = iris.coords.DimCoord( + [182.5, 547.5], + standard_name='time', + bounds=[[0, 365], [365, 730]], + units='days since 1950-01-01 00:00') + self.coords_spec = [(self.lats, 0), (self.lons, 1)] + self.fx_area = iris.cube.Cube(fx_area_data, + dim_coords_and_dims=self.coords_spec) + self.fx_volume = iris.cube.Cube(fx_volume_data, + dim_coords_and_dims=[ + (self.depth, 0), + (self.lats, 1), + (self.lons, 2) + ]) + self.monthly_volume = iris.cube.Cube(np.ones((12, 3, 3, 3)), + dim_coords_and_dims=[ + (self.monthly_times, 0), + (self.depth, 1), + (self.lats, 2), + (self.lons, 3) + ]) + + @pytest.mark.parametrize('var_name', ['areacella', 'areacello']) + def test_add_cell_measure_area(self, var_name): + """Test add area fx variables as cell measures.""" + self.fx_area.var_name = var_name + self.fx_area.standard_name = 'cell_area' + self.fx_area.units = 'm2' + cube = iris.cube.Cube(self.new_cube_data, + dim_coords_and_dims=self.coords_spec) + cube = add_supplementary_variables(cube, [self.fx_area]) + assert cube.cell_measure(self.fx_area.standard_name) is not None + + def test_add_cell_measure_volume(self): + """Test add volume as cell measure.""" + self.fx_volume.var_name = 'volcello' + self.fx_volume.standard_name = 'ocean_volume' + self.fx_volume.units = 'm3' + cube = iris.cube.Cube(self.new_cube_3D_data, + dim_coords_and_dims=[ + (self.depth, 0), + (self.lats, 1), + (self.lons, 2)]) + cube = add_supplementary_variables(cube, [self.fx_volume]) + assert cube.cell_measure(self.fx_volume.standard_name) is not None + + def test_no_cell_measure(self): + """Test no cell measure is added.""" + cube = iris.cube.Cube(self.new_cube_3D_data, + dim_coords_and_dims=[ + (self.depth, 0), + (self.lats, 1), + (self.lons, 2)]) + cube = add_supplementary_variables(cube, []) + assert cube.cell_measures() == [] + + def test_add_supplementary_vars(self): + """Test invalid variable is not added as cell measure.""" + self.fx_area.var_name = 'sftlf' + self.fx_area.standard_name = "land_area_fraction" + self.fx_area.units = '%' + cube = iris.cube.Cube(self.new_cube_data, + dim_coords_and_dims=self.coords_spec) + cube = add_supplementary_variables(cube, [self.fx_area]) + assert cube.ancillary_variable(self.fx_area.standard_name) is not None + + @pytest.mark.parametrize('use_legacy_supplementaries', [True, False]) + def test_wrong_shape(self, use_legacy_supplementaries, monkeypatch): + """Test variable is not added if it's not broadcastable to cube.""" + monkeypatch.setitem( + esmvalcore.config.CFG, + 'use_legacy_supplementaries', + use_legacy_supplementaries, + ) + volume_data = np.ones((2, 3, 3, 3)) + volume_cube = iris.cube.Cube( + volume_data, + dim_coords_and_dims=[(self.yearly_times, 0), + (self.depth, 1), + (self.lats, 2), + (self.lons, 3)]) + volume_cube.standard_name = 'ocean_volume' + volume_cube.var_name = 'volcello' + volume_cube.units = 'm3' + data = np.ones((12, 3, 3, 3)) + cube = iris.cube.Cube( + data, + dim_coords_and_dims=[(self.monthly_times, 0), + (self.depth, 1), + (self.lats, 2), + (self.lons, 3)]) + cube.var_name = 'thetao' + if use_legacy_supplementaries: + add_supplementary_variables(cube, [volume_cube]) + assert cube.cell_measures() == [] + else: + with pytest.raises(iris.exceptions.CannotAddError): + add_supplementary_variables(cube, [volume_cube]) + + def test_remove_supplementary_vars(self): + """Test supplementary variables are removed from cube.""" + cube = iris.cube.Cube(self.new_cube_3D_data, + dim_coords_and_dims=[(self.depth, 0), + (self.lats, 1), + (self.lons, 2)]) + self.fx_area.var_name = 'areacella' + self.fx_area.standard_name = 'cell_area' + self.fx_area.units = 'm2' + add_cell_measure(cube, self.fx_area, measure='area') + assert cube.cell_measure(self.fx_area.standard_name) is not None + self.fx_area.var_name = 'sftlf' + self.fx_area.standard_name = "land_area_fraction" + self.fx_area.units = '%' + add_ancillary_variable(cube, self.fx_area) + assert cube.ancillary_variable(self.fx_area.standard_name) is not None + cube = remove_supplementary_variables(cube) + assert cube.cell_measures() == [] + assert cube.ancillary_variables() == [] diff --git a/tests/integration/preprocessor/_supplementary_vars/test_register.py b/tests/integration/preprocessor/_supplementary_vars/test_register.py new file mode 100644 index 0000000000..cfe6d5b7da --- /dev/null +++ b/tests/integration/preprocessor/_supplementary_vars/test_register.py @@ -0,0 +1,39 @@ +import pytest + +from esmvalcore.preprocessor import _supplementary_vars + + +def test_register(monkeypatch): + """Test registering an supplementary variable.""" + registered = {} + monkeypatch.setattr( + _supplementary_vars, + 'PREPROCESSOR_SUPPLEMENTARIES', + registered, + ) + + @_supplementary_vars.register_supplementaries( + ['areacella'], + required='require_at_least_one', + ) + def test_func(): + pass + + assert registered == { + 'test_func': { + 'required': 'require_at_least_one', + 'variables': ['areacella'], + } + } + + +def test_register_invalid_fails(): + """test that registering an invalid requirement fails.""" + with pytest.raises(NotImplementedError): + + @_supplementary_vars.register_supplementaries( + ['areacella'], + required='invalid', + ) + def test_func(): + pass diff --git a/tests/integration/preprocessor/test_preprocessing_task.py b/tests/integration/preprocessor/test_preprocessing_task.py new file mode 100644 index 0000000000..99d971cf8c --- /dev/null +++ b/tests/integration/preprocessor/test_preprocessing_task.py @@ -0,0 +1,42 @@ +"""Tests for `esmvalcore.preprocessor.PreprocessingTask`.""" +import iris +import iris.cube +from prov.model import ProvDocument + +from esmvalcore.dataset import Dataset +from esmvalcore.preprocessor import PreprocessingTask, PreprocessorFile + + +def test_load_save_task(tmp_path): + """Test that a task that just loads and saves a file.""" + # Prepare a test dataset + cube = iris.cube.Cube(data=[273.], var_name='tas', units='K') + in_file = tmp_path / 'tas_in.nc' + iris.save(cube, in_file) + dataset = Dataset(short_name='tas') + dataset.files = [in_file] + dataset._load_with_callback = lambda _: cube + + # Create task + task = PreprocessingTask([ + PreprocessorFile( + filename=tmp_path / 'tas_out.nc', + settings={}, + datasets=[dataset], + ), + ]) + + # Create an 'activity' representing a run of the tool + provenance = ProvDocument() + provenance.add_namespace('software', uri='https://example.com/software') + activity = provenance.activity('software:esmvalcore') + task.initialize_provenance(activity) + + task.run() + + assert len(task.products) == 1 + preproc_file = task.products.pop().filename + result = iris.load_cube(preproc_file) + + result.attributes.clear() + assert result == cube diff --git a/tests/integration/recipe/test_check.py b/tests/integration/recipe/test_check.py index 658382d538..367983a0cc 100644 --- a/tests/integration/recipe/test_check.py +++ b/tests/integration/recipe/test_check.py @@ -7,8 +7,10 @@ import pyesgf.search.results import pytest +import esmvalcore._recipe.check import esmvalcore.esgf from esmvalcore._recipe import check +from esmvalcore.dataset import Dataset from esmvalcore.exceptions import RecipeError from esmvalcore.preprocessor import PreprocessorFile @@ -52,16 +54,16 @@ @mock.patch('esmvalcore._recipe.check.logger', autospec=True) def test_data_availability_data(mock_logger, input_files, var, error): """Test check for data when data is present.""" - saved_var = dict(var) - input_files = [Path(f) for f in input_files] + dataset = Dataset(**var) + dataset.files = [Path(f) for f in input_files] if error is None: - check.data_availability(input_files, var, None, None) + check.data_availability(dataset) mock_logger.error.assert_not_called() else: with pytest.raises(RecipeError) as rec_err: - check.data_availability(input_files, var, None, None) + check.data_availability(dataset) assert str(rec_err.value) == error - assert var == saved_var + assert dataset.facets == var DATA_AVAILABILITY_NO_DATA: List[Any] = [ @@ -79,8 +81,7 @@ def test_data_availability_data(mock_logger, input_files, var, error): @mock.patch('esmvalcore._recipe.check.logger', autospec=True) def test_data_availability_no_data(mock_logger, dirnames, filenames, error): """Test check for data when no data is present.""" - var = dict(VAR) - var_no_filename = { + facets = { 'frequency': 'mon', 'short_name': 'tas', 'timerange': '2020/2025', @@ -88,14 +89,16 @@ def test_data_availability_no_data(mock_logger, dirnames, filenames, error): 'start_year': 2020, 'end_year': 2025 } - patterns = [ + dataset = Dataset(**facets) + dataset.files = [] + dataset._file_globs = [ os.path.join(d, f) for d in dirnames for f in filenames ] - error_first = ('No input files found for variable %s', var_no_filename) + error_first = ('No input files found for %s', dataset) error_last = ("Set 'log_level' to 'debug' to get more information", ) with pytest.raises(RecipeError) as rec_err: - check.data_availability([], var, patterns) - assert str(rec_err.value) == 'Missing data for alias: tas' + check.data_availability(dataset) + assert str(rec_err.value) == 'Missing data for Dataset: tas' if error is None: assert mock_logger.error.call_count == 2 errors = [error_first, error_last] @@ -104,7 +107,7 @@ def test_data_availability_no_data(mock_logger, dirnames, filenames, error): errors = [error_first, error, error_last] calls = [mock.call(*e) for e in errors] assert mock_logger.error.call_args_list == calls - assert var == VAR + assert dataset.facets == facets GOOD_TIMERANGES = [ @@ -159,6 +162,34 @@ def test_valid_time_selection_rejections(timerange, message): assert str(rec_err.value) == message +def test_differing_timeranges(caplog): + timeranges = set() + timeranges.add('1950/1951') + timeranges.add('1950/1952') + required_variables = [ + { + 'short_name': 'rsdscs', + 'timerange': '1950/1951' + }, + { + 'short_name': 'rsuscs', + 'timerange': '1950/1952' + }, + ] + with pytest.raises(ValueError) as exc: + check.differing_timeranges( + timeranges, required_variables) + expected_log = ( + f"Differing timeranges with values {timeranges} " + "found for required variables " + "[{'short_name': 'rsdscs', 'timerange': '1950/1951'}, " + "{'short_name': 'rsuscs', 'timerange': '1950/1952'}]. " + "Set `timerange` to a common value." + ) + + assert expected_log in str(exc.value) + + def test_data_availability_nonexistent(tmp_path): var = { 'dataset': 'ABC', @@ -180,15 +211,17 @@ def test_data_availability_nonexistent(tmp_path): ) dest_folder = tmp_path input_files = [esmvalcore.esgf.ESGFFile([result]).local_file(dest_folder)] - check.data_availability(input_files, var, patterns=[]) + dataset = Dataset(**var) + dataset.files = input_files + check.data_availability(dataset) def test_reference_for_bias_preproc_empty(): """Test ``reference_for_bias_preproc``.""" products = { - PreprocessorFile({'filename': 10}, {}), - PreprocessorFile({'filename': 20}, {}), - PreprocessorFile({'filename': 30}, {'trend': {}}), + PreprocessorFile(filename=10), + PreprocessorFile(filename=20), + PreprocessorFile(filename=30), } check.reference_for_bias_preproc(products) @@ -196,11 +229,14 @@ def test_reference_for_bias_preproc_empty(): def test_reference_for_bias_preproc_one_ref(): """Test ``reference_for_bias_preproc`` with one reference.""" products = { - PreprocessorFile({'filename': 90}, {}), - PreprocessorFile({'filename': 10}, {'bias': {}}), - PreprocessorFile({'filename': 20}, {'bias': {}}), - PreprocessorFile({'filename': 30, 'reference_for_bias': True}, - {'bias': {}}) + PreprocessorFile(filename=90), + PreprocessorFile(filename=10, + settings={'bias': {}}), + PreprocessorFile(filename=20, + settings={'bias': {}}), + PreprocessorFile(filename=30, + settings={'bias': {}}, + attributes={'reference_for_bias': True}), } check.reference_for_bias_preproc(products) @@ -208,10 +244,13 @@ def test_reference_for_bias_preproc_one_ref(): def test_reference_for_bias_preproc_no_ref(): """Test ``reference_for_bias_preproc`` with no reference.""" products = { - PreprocessorFile({'filename': 90}, {}), - PreprocessorFile({'filename': 10}, {'bias': {}}), - PreprocessorFile({'filename': 20}, {'bias': {}}), - PreprocessorFile({'filename': 30}, {'bias': {}}) + PreprocessorFile(filename=90), + PreprocessorFile(filename=10, + settings={'bias': {}}), + PreprocessorFile(filename=20, + settings={'bias': {}}), + PreprocessorFile(filename=30, + settings={'bias': {}}) } with pytest.raises(RecipeError) as rec_err: check.reference_for_bias_preproc(products) @@ -231,12 +270,14 @@ def test_reference_for_bias_preproc_no_ref(): def test_reference_for_bias_preproc_two_refs(): """Test ``reference_for_bias_preproc`` with two references.""" products = { - PreprocessorFile({'filename': 90}, {}), - PreprocessorFile({'filename': 10}, {'bias': {}}), - PreprocessorFile({'filename': 20, 'reference_for_bias': True}, - {'bias': {}}), - PreprocessorFile({'filename': 30, 'reference_for_bias': True}, - {'bias': {}}) + PreprocessorFile(filename=90), + PreprocessorFile(filename=10, settings={'bias': {}}), + PreprocessorFile(filename=20, + attributes={'reference_for_bias': True}, + settings={'bias': {}}), + PreprocessorFile(filename=30, + attributes={'reference_for_bias': True}, + settings={'bias': {}}) } with pytest.raises(RecipeError) as rec_err: check.reference_for_bias_preproc(products) diff --git a/tests/integration/recipe/test_recipe.py b/tests/integration/recipe/test_recipe.py index df5f3c5443..5739768f90 100644 --- a/tests/integration/recipe/test_recipe.py +++ b/tests/integration/recipe/test_recipe.py @@ -1,31 +1,32 @@ import os +import re from collections import defaultdict -from copy import deepcopy from pathlib import Path from pprint import pformat from textwrap import dedent -from unittest.mock import create_autospec, patch, sentinel +from unittest.mock import create_autospec import iris import pytest import yaml -from nested_lookup import get_occurrence_of_value, nested_update +from nested_lookup import get_occurrence_of_value from PIL import Image import esmvalcore +import esmvalcore._task from esmvalcore._recipe.recipe import ( - TASKSEP, - _dataset_to_file, - _get_derive_input_variables, + _get_input_datasets, + _representative_dataset, read_recipe_file, ) from esmvalcore._task import DiagnosticTask -from esmvalcore.cmor.check import CheckLevels +from esmvalcore.config import Session +from esmvalcore.config._config import TASKSEP from esmvalcore.config._diagnostics import TAGS -from esmvalcore.exceptions import InputFilesNotFound, RecipeError +from esmvalcore.dataset import Dataset +from esmvalcore.exceptions import RecipeError +from esmvalcore.local import _get_output_file from esmvalcore.preprocessor import DEFAULT_ORDER, PreprocessingTask -from esmvalcore.preprocessor._io import concatenate_callback - from tests.integration.test_provenance import check_provenance TAGS_FOR_TESTING = { @@ -59,7 +60,6 @@ MANDATORY_DATASET_KEYS = ( 'dataset', 'diagnostic', - 'filename', 'frequency', 'institute', 'long_name', @@ -82,32 +82,15 @@ ) DEFAULT_PREPROCESSOR_STEPS = ( - 'add_fx_variables', - 'cleanup', - 'cmor_check_data', - 'cmor_check_metadata', - 'concatenate', - 'clip_timerange', - 'fix_data', - 'fix_file', - 'fix_metadata', 'load', - 'remove_fx_variables', + 'cleanup', + 'remove_supplementary_variables', 'save', ) INITIALIZATION_ERROR_MSG = 'Could not create all tasks' -@pytest.fixture -def config_user(session): - cfg = session.to_config_user() - cfg['offline'] = True - cfg['check_level'] = CheckLevels.DEFAULT - cfg['diagnostics'] = set() - return cfg - - def create_test_file(filename, tracking_id=None): dirname = os.path.dirname(filename) if not os.path.exists(dirname): @@ -121,109 +104,13 @@ def create_test_file(filename, tracking_id=None): iris.save(cube, filename) -def _get_default_settings_for_chl(fix_dir, save_filename, preprocessor): +def _get_default_settings_for_chl(fix_dir, save_filename): """Get default preprocessor settings for chl.""" - standard_name = ('mass_concentration_of_phytoplankton_' - 'expressed_as_chlorophyll_in_sea_water') defaults = { 'load': { - 'callback': concatenate_callback, - }, - 'concatenate': {}, - 'fix_file': { - 'alias': 'CanESM2', - 'dataset': 'CanESM2', - 'diagnostic': 'diagnostic_name', - 'ensemble': 'r1i1p1', - 'exp': 'historical', - 'filename': Path(fix_dir.replace('_fixed', '.nc')), - 'frequency': 'yr', - 'institute': ['CCCma'], - 'long_name': 'Total Chlorophyll Mass Concentration', - 'mip': 'Oyr', - 'modeling_realm': ['ocnBgchem'], - 'original_short_name': 'chl', - 'output_dir': fix_dir, - 'preprocessor': preprocessor, - 'product': ['output1', 'output2'], - 'project': 'CMIP5', - 'recipe_dataset_index': 0, - 'short_name': 'chl', - 'standard_name': standard_name, - 'timerange': '2000/2005', - 'units': 'kg m-3', - 'variable_group': 'chl', - }, - 'fix_data': { - 'check_level': CheckLevels.DEFAULT, - 'alias': 'CanESM2', - 'dataset': 'CanESM2', - 'diagnostic': 'diagnostic_name', - 'ensemble': 'r1i1p1', - 'exp': 'historical', - 'filename': Path(fix_dir.replace('_fixed', '.nc')), - 'frequency': 'yr', - 'institute': ['CCCma'], - 'long_name': 'Total Chlorophyll Mass Concentration', - 'mip': 'Oyr', - 'modeling_realm': ['ocnBgchem'], - 'original_short_name': 'chl', - 'preprocessor': preprocessor, - 'product': ['output1', 'output2'], - 'project': 'CMIP5', - 'recipe_dataset_index': 0, - 'short_name': 'chl', - 'standard_name': standard_name, - 'timerange': '2000/2005', - 'units': 'kg m-3', - 'variable_group': 'chl', - }, - 'fix_metadata': { - 'check_level': CheckLevels.DEFAULT, - 'alias': 'CanESM2', - 'dataset': 'CanESM2', - 'diagnostic': 'diagnostic_name', - 'ensemble': 'r1i1p1', - 'exp': 'historical', - 'filename': Path(fix_dir.replace('_fixed', '.nc')), - 'frequency': 'yr', - 'institute': ['CCCma'], - 'long_name': 'Total Chlorophyll Mass Concentration', - 'mip': 'Oyr', - 'modeling_realm': ['ocnBgchem'], - 'original_short_name': 'chl', - 'preprocessor': preprocessor, - 'product': ['output1', 'output2'], - 'project': 'CMIP5', - 'recipe_dataset_index': 0, - 'short_name': 'chl', - 'standard_name': standard_name, - 'timerange': '2000/2005', - 'units': 'kg m-3', - 'variable_group': 'chl', - }, - 'clip_timerange': { - 'timerange': '2000/2005', - }, - 'cmor_check_metadata': { - 'check_level': CheckLevels.DEFAULT, - 'cmor_table': 'CMIP5', - 'mip': 'Oyr', - 'short_name': 'chl', - 'frequency': 'yr', - }, - 'cmor_check_data': { - 'check_level': CheckLevels.DEFAULT, - 'cmor_table': 'CMIP5', - 'mip': 'Oyr', - 'short_name': 'chl', - 'frequency': 'yr', - }, - 'add_fx_variables': { - 'fx_variables': {}, - 'check_level': CheckLevels.DEFAULT, + 'callback': 'default' }, - 'remove_fx_variables': {}, + 'remove_supplementary_variables': {}, 'cleanup': { 'remove': [fix_dir] }, @@ -253,7 +140,7 @@ def get_required(short_name, _): return required monkeypatch.setattr( - esmvalcore._recipe.recipe, + esmvalcore._recipe.to_datasets, 'get_required', get_required, ) @@ -273,31 +160,24 @@ def get_required(short_name, _): """) -def get_recipe(tempdir, content, cfg): +def get_recipe(tempdir: Path, content: str, session: Session): """Save and load recipe content.""" recipe_file = tempdir / 'recipe_test.yml' # Add mandatory documentation section content = str(DEFAULT_DOCUMENTATION + content) recipe_file.write_text(content) - recipe = read_recipe_file(str(recipe_file), cfg) + recipe = read_recipe_file(recipe_file, session) return recipe -def test_recipe_no_datasets(tmp_path, config_user): +def test_recipe_no_datasets(tmp_path, session): content = dedent(""" - preprocessors: - preprocessor_name: - extract_levels: - levels: 85000 - scheme: nearest - diagnostics: diagnostic_name: variables: ta: - preprocessor: preprocessor_name project: CMIP5 mip: Amon exp: historical @@ -308,18 +188,61 @@ def test_recipe_no_datasets(tmp_path, config_user): """) exc_message = ("You have not specified any dataset " "or additional_dataset groups for variable " - "{'preprocessor': 'preprocessor_name', 'project': 'CMIP5'," - " 'mip': 'Amon', 'exp': 'historical', 'ensemble': 'r1i1p1'" - ", 'start_year': 1999, 'end_year': 2002, 'variable_group':" - " 'ta', 'short_name': 'ta', 'diagnostic': " - "'diagnostic_name'} Exiting.") + "ta in diagnostic diagnostic_name.") with pytest.raises(RecipeError) as exc: - get_recipe(tmp_path, content, config_user) + get_recipe(tmp_path, content, session) assert str(exc.value) == exc_message -def test_simple_recipe(tmp_path, patched_datafinder, config_user): - script = tmp_path / 'diagnostic.py' +@pytest.mark.parametrize('skip_nonexistent', [True, False]) +def test_recipe_no_data(tmp_path, session, skip_nonexistent): + content = dedent(""" + datasets: + - dataset: GFDL-ESM2G + + diagnostics: + diagnostic_name: + variables: + ta: + project: CMIP5 + mip: Amon + exp: historical + ensemble: r1i1p1 + start_year: 1999 + end_year: 2002 + scripts: null + """) + session['skip_nonexistent'] = skip_nonexistent + with pytest.raises(RecipeError) as error: + get_recipe(tmp_path, content, session) + if skip_nonexistent: + msg = ("Did not find any input data for task diagnostic_name/ta") + else: + msg = ("Missing data for preprocessor diagnostic_name/ta:\n" + "- Missing data for Dataset: .*") + assert re.match(msg, error.value.failed_tasks[0].message) + + +@pytest.mark.parametrize('script_file', ['diagnostic.py', 'diagnostic.ncl']) +def test_simple_recipe( + tmp_path, + patched_datafinder, + session, + script_file, + monkeypatch, +): + + def ncl_version(): + return '6.5' + + monkeypatch.setattr(esmvalcore._recipe.check, 'ncl_version', ncl_version) + + def which(interpreter): + return interpreter + + monkeypatch.setattr(esmvalcore._task, 'which', which) + + script = tmp_path / script_file script.write_text('') content = dedent(""" datasets: @@ -342,8 +265,7 @@ def test_simple_recipe(tmp_path, patched_datafinder, config_user): mip: Amon exp: historical ensemble: r1i1p1 - start_year: 1999 - end_year: 2002 + timerange: 1999/2002 additional_datasets: - dataset: MPI-ESM-LR scripts: @@ -352,29 +274,15 @@ def test_simple_recipe(tmp_path, patched_datafinder, config_user): custom_setting: 1 """.format(script)) - recipe = get_recipe(tmp_path, content, config_user) - raw = yaml.safe_load(content) - # Perform some sanity checks on recipe expansion/normalization - print("Expanded recipe:") - assert len(recipe.diagnostics) == len(raw['diagnostics']) - for diagnostic_name, diagnostic in recipe.diagnostics.items(): - print(pformat(diagnostic)) - source = raw['diagnostics'][diagnostic_name] - - # Check that 'variables' have been read and updated - assert len(diagnostic['preprocessor_output']) == len( - source['variables']) - for variable_name, variables in diagnostic[ - 'preprocessor_output'].items(): - assert len(variables) == 3 - for variable in variables: - for key in MANDATORY_DATASET_KEYS: - assert key in variable and variable[key] - assert variable_name == variable['short_name'] + recipe = get_recipe(tmp_path, content, session) + # Check that datasets have been read and updated + assert len(recipe.datasets) == 3 + for dataset in recipe.datasets: + for key in MANDATORY_DATASET_KEYS: + assert key in dataset.facets and dataset.facets[key] # Check that the correct tasks have been created - variables = recipe.diagnostics['diagnostic_name']['preprocessor_output'][ - 'ta'] + datasets = recipe.datasets tasks = {t for task in recipe.tasks for t in task.flatten()} preproc_tasks = {t for t in tasks if isinstance(t, PreprocessingTask)} diagnostic_tasks = {t for t in tasks if isinstance(t, DiagnosticTask)} @@ -384,13 +292,19 @@ def test_simple_recipe(tmp_path, patched_datafinder, config_user): print("Task", task.name) assert task.order == list(DEFAULT_ORDER) for product in task.products: - variable = [ - v for v in variables if v['filename'] == product.filename + dataset = [ + d for d in datasets if _get_output_file( + d.facets, session.preproc_dir) == product.filename ][0] - assert product.attributes == variable + assert product.datasets == [dataset] + attributes = dict(dataset.facets) + attributes['filename'] = product.filename + attributes['start_year'] = 1999 + attributes['end_year'] = 2002 + assert product.attributes == attributes for step in DEFAULT_PREPROCESSOR_STEPS: assert step in product.settings - assert len(product.files) == 2 + assert len(dataset.files) == 2 assert len(diagnostic_tasks) == 1 for task in diagnostic_tasks: @@ -400,11 +314,13 @@ def test_simple_recipe(tmp_path, patched_datafinder, config_user): for key in MANDATORY_SCRIPT_SETTINGS_KEYS: assert key in task.settings and task.settings[key] assert task.settings['custom_setting'] == 1 - # Filled recipe is not created as there are no wildcards. - assert recipe._updated_recipe == {} + + # Check that NCL interface is enabled for NCL scripts. + write_ncl_interface = script.suffix == '.ncl' + assert datasets[0].session['write_ncl_interface'] == write_ncl_interface -def test_simple_recipe_fill(tmp_path, patched_datafinder, config_user): +def test_write_filled_recipe(tmp_path, patched_datafinder, session): script = tmp_path / 'diagnostic.py' script.write_text('') content = dedent(""" @@ -438,19 +354,24 @@ def test_simple_recipe_fill(tmp_path, patched_datafinder, config_user): custom_setting: 1 """.format(script)) - recipe = get_recipe(tmp_path, content, config_user) - preprocessor_output = recipe.diagnostics['diagnostic_name'][ - 'preprocessor_output'] - recipe._fill_wildcards('ta', preprocessor_output) - assert recipe._updated_recipe - assert get_occurrence_of_value(recipe._updated_recipe, value='*') == 0 - assert get_occurrence_of_value(recipe._updated_recipe, - value='1990/2019') == 2 - assert get_occurrence_of_value(recipe._updated_recipe, - value='1990/P2Y') == 1 + recipe = get_recipe(tmp_path, content, session) + + session.run_dir.mkdir(parents=True) + esmvalcore._recipe.recipe.Recipe.write_filled_recipe(recipe) + + recipe_file = session.run_dir / 'recipe_test_filled.yml' + assert recipe_file.is_file() + + updated_recipe_object = read_recipe_file(recipe_file, session) + updated_recipe = updated_recipe_object._raw_recipe + print(pformat(updated_recipe)) + assert get_occurrence_of_value(updated_recipe, value='*') == 0 + assert get_occurrence_of_value(updated_recipe, value='1990/2019') == 2 + assert get_occurrence_of_value(updated_recipe, value='1990/P2Y') == 1 + assert len(updated_recipe_object.datasets) == 3 -def test_fx_preproc_error(tmp_path, patched_datafinder, config_user): +def test_fx_preproc_error(tmp_path, patched_datafinder, session): script = tmp_path / 'diagnostic.py' script.write_text('') content = dedent(""" @@ -480,13 +401,12 @@ def test_fx_preproc_error(tmp_path, patched_datafinder, config_user): msg = ("Time coordinate preprocessor step(s) ['extract_season'] not " "permitted on fx vars, please remove them from recipe") with pytest.raises(Exception) as rec_err_exp: - get_recipe(tmp_path, content, config_user) + get_recipe(tmp_path, content, session) assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG assert str(rec_err_exp.value.failed_tasks[0].message) == msg -def test_default_preprocessor(tmp_path, patched_datafinder, config_user): - +def test_default_preprocessor(tmp_path, patched_datafinder, session): content = dedent(""" diagnostics: diagnostic_name: @@ -503,7 +423,7 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user): scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) assert len(recipe.tasks) == 1 task = recipe.tasks.pop() @@ -514,13 +434,12 @@ def test_default_preprocessor(tmp_path, patched_datafinder, config_user): fix_dir = os.path.join( preproc_dir, 'CMIP5_CanESM2_Oyr_historical_r1i1p1_chl_2000-2005_fixed') - defaults = _get_default_settings_for_chl(fix_dir, product.filename, - 'default') + defaults = _get_default_settings_for_chl(fix_dir, product.filename) assert product.settings == defaults def test_default_preprocessor_custom_order(tmp_path, patched_datafinder, - config_user): + session): """Test if default settings are used when ``custom_order`` is ``True``.""" content = dedent(""" @@ -544,7 +463,7 @@ def test_default_preprocessor_custom_order(tmp_path, patched_datafinder, scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) assert len(recipe.tasks) == 1 task = recipe.tasks.pop() @@ -555,13 +474,73 @@ def test_default_preprocessor_custom_order(tmp_path, patched_datafinder, fix_dir = os.path.join( preproc_dir, 'CMIP5_CanESM2_Oyr_historical_r1i1p1_chl_2000-2005_fixed') - defaults = _get_default_settings_for_chl(fix_dir, product.filename, - 'default_custom_order') + defaults = _get_default_settings_for_chl(fix_dir, product.filename) assert product.settings == defaults -def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): +def test_invalid_preprocessor(tmp_path, patched_datafinder, session): + """Test the error message when the named prepreprocesor is not defined.""" + content = dedent(""" + diagnostics: + diagnostic_name: + variables: + chl: + preprocessor: not_defined + project: CMIP5 + mip: Oyr + exp: historical + start_year: 2000 + end_year: 2005 + ensemble: r1i1p1 + additional_datasets: + - {dataset: CanESM2} + scripts: null + """) + + with pytest.raises(RecipeError) as error: + get_recipe(tmp_path, content, session) + msg = "Unknown preprocessor 'not_defined' in .*" + assert re.match(msg, error.value.failed_tasks[0].message) + + +def test_disable_preprocessor_function(tmp_path, patched_datafinder, session): + """Test if default settings are used when ``custom_order`` is ``True``.""" + + content = dedent(""" + datasets: + - dataset: HadGEM3-GC31-LL + ensemble: r1i1p1f1 + exp: historical + grid: gn + + preprocessors: + keep_supplementaries: + remove_supplementary_variables: False + + diagnostics: + diagnostic_name: + variables: + tas: + preprocessor: keep_supplementaries + project: CMIP6 + mip: Amon + timerange: 2000/2005 + supplementaries: + - short_name: areacella + mip: fx + scripts: null + """) + + recipe = get_recipe(tmp_path, content, session) + assert len(recipe.tasks) == 1 + task = recipe.tasks.pop() + assert len(task.products) == 1 + product = task.products.pop() + assert 'remove_supplementary_variables' not in product.settings + + +def test_default_fx_preprocessor(tmp_path, patched_datafinder, session): content = dedent(""" diagnostics: diagnostic_name: @@ -576,7 +555,7 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) assert len(recipe.tasks) == 1 task = recipe.tasks.pop() @@ -590,97 +569,9 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): defaults = { 'load': { - 'callback': concatenate_callback, - }, - 'concatenate': {}, - 'fix_file': { - 'alias': 'CanESM2', - 'dataset': 'CanESM2', - 'diagnostic': 'diagnostic_name', - 'ensemble': 'r0i0p0', - 'exp': 'historical', - 'filename': Path(fix_dir.replace('_fixed', '.nc')), - 'frequency': 'fx', - 'institute': ['CCCma'], - 'long_name': 'Land Area Fraction', - 'mip': 'fx', - 'modeling_realm': ['atmos'], - 'original_short_name': 'sftlf', - 'output_dir': fix_dir, - 'preprocessor': 'default', - 'product': ['output1', 'output2'], - 'project': 'CMIP5', - 'recipe_dataset_index': 0, - 'short_name': 'sftlf', - 'standard_name': 'land_area_fraction', - 'units': '%', - 'variable_group': 'sftlf' - }, - 'fix_data': { - 'check_level': CheckLevels.DEFAULT, - 'alias': 'CanESM2', - 'dataset': 'CanESM2', - 'diagnostic': 'diagnostic_name', - 'ensemble': 'r0i0p0', - 'exp': 'historical', - 'filename': Path(fix_dir.replace('_fixed', '.nc')), - 'frequency': 'fx', - 'institute': ['CCCma'], - 'long_name': 'Land Area Fraction', - 'mip': 'fx', - 'modeling_realm': ['atmos'], - 'original_short_name': 'sftlf', - 'preprocessor': 'default', - 'product': ['output1', 'output2'], - 'project': 'CMIP5', - 'recipe_dataset_index': 0, - 'short_name': 'sftlf', - 'standard_name': 'land_area_fraction', - 'units': '%', - 'variable_group': 'sftlf' - }, - 'fix_metadata': { - 'check_level': CheckLevels.DEFAULT, - 'alias': 'CanESM2', - 'dataset': 'CanESM2', - 'diagnostic': 'diagnostic_name', - 'ensemble': 'r0i0p0', - 'exp': 'historical', - 'filename': Path(fix_dir.replace('_fixed', '.nc')), - 'frequency': 'fx', - 'institute': ['CCCma'], - 'long_name': 'Land Area Fraction', - 'mip': 'fx', - 'modeling_realm': ['atmos'], - 'original_short_name': 'sftlf', - 'preprocessor': 'default', - 'product': ['output1', 'output2'], - 'project': 'CMIP5', - 'recipe_dataset_index': 0, - 'short_name': 'sftlf', - 'standard_name': 'land_area_fraction', - 'units': '%', - 'variable_group': 'sftlf' - }, - 'cmor_check_metadata': { - 'check_level': CheckLevels.DEFAULT, - 'cmor_table': 'CMIP5', - 'mip': 'fx', - 'short_name': 'sftlf', - 'frequency': 'fx', - }, - 'cmor_check_data': { - 'check_level': CheckLevels.DEFAULT, - 'cmor_table': 'CMIP5', - 'mip': 'fx', - 'short_name': 'sftlf', - 'frequency': 'fx', - }, - 'add_fx_variables': { - 'fx_variables': {}, - 'check_level': CheckLevels.DEFAULT, + 'callback': 'default' }, - 'remove_fx_variables': {}, + 'remove_supplementary_variables': {}, 'cleanup': { 'remove': [fix_dir] }, @@ -692,7 +583,7 @@ def test_default_fx_preprocessor(tmp_path, patched_datafinder, config_user): assert product.settings == defaults -def test_empty_variable(tmp_path, patched_datafinder, config_user): +def test_empty_variable(tmp_path, patched_datafinder, session): """Test that it is possible to specify all information in the dataset.""" content = dedent(""" diagnostics: @@ -710,7 +601,7 @@ def test_empty_variable(tmp_path, patched_datafinder, config_user): scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) assert len(recipe.tasks) == 1 task = recipe.tasks.pop() assert len(task.products) == 1 @@ -719,202 +610,6 @@ def test_empty_variable(tmp_path, patched_datafinder, config_user): assert product.attributes['dataset'] == 'CanESM2' -def test_cmip3_variable_autocomplete(tmp_path, patched_datafinder, - config_user): - """Test that required information is automatically added for CMIP5.""" - content = dedent(""" - diagnostics: - test: - additional_datasets: - - dataset: bccr_bcm2_0 - project: CMIP3 - mip: A1 - frequency: mon - exp: historical - start_year: 2000 - end_year: 2001 - ensemble: r1i1p1 - modeling_realm: atmos - variables: - zg: - scripts: null - """) - - recipe = get_recipe(tmp_path, content, config_user) - variable = recipe.diagnostics['test']['preprocessor_output']['zg'][0] - - reference = { - 'dataset': 'bccr_bcm2_0', - 'diagnostic': 'test', - 'ensemble': 'r1i1p1', - 'exp': 'historical', - 'frequency': 'mon', - 'institute': ['BCCR'], - 'long_name': 'Geopotential Height', - 'mip': 'A1', - 'modeling_realm': 'atmos', - 'preprocessor': 'default', - 'project': 'CMIP3', - 'short_name': 'zg', - 'standard_name': 'geopotential_height', - 'timerange': '2000/2001', - 'units': 'm', - } - for key in reference: - assert variable[key] == reference[key] - - -def test_cmip5_variable_autocomplete(tmp_path, patched_datafinder, - config_user): - """Test that required information is automatically added for CMIP5.""" - content = dedent(""" - diagnostics: - test: - additional_datasets: - - dataset: CanESM2 - project: CMIP5 - mip: 3hr - exp: historical - start_year: 2000 - end_year: 2001 - ensemble: r1i1p1 - variables: - pr: - scripts: null - """) - - recipe = get_recipe(tmp_path, content, config_user) - variable = recipe.diagnostics['test']['preprocessor_output']['pr'][0] - - reference = { - 'dataset': 'CanESM2', - 'diagnostic': 'test', - 'ensemble': 'r1i1p1', - 'exp': 'historical', - 'frequency': '3hr', - 'institute': ['CCCma'], - 'long_name': 'Precipitation', - 'mip': '3hr', - 'modeling_realm': ['atmos'], - 'preprocessor': 'default', - 'project': 'CMIP5', - 'short_name': 'pr', - 'standard_name': 'precipitation_flux', - 'timerange': '2000/2001', - 'units': 'kg m-2 s-1', - } - for key in reference: - assert variable[key] == reference[key] - - -def test_cmip6_variable_autocomplete(tmp_path, patched_datafinder, - config_user): - """Test that required information is automatically added for CMIP6.""" - content = dedent(""" - diagnostics: - test: - additional_datasets: - - dataset: HadGEM3-GC31-LL - project: CMIP6 - mip: 3hr - exp: historical - start_year: 2000 - end_year: 2001 - ensemble: r2i1p1f1 - grid: gn - variables: - pr: - scripts: null - """) - - recipe = get_recipe(tmp_path, content, config_user) - variable = recipe.diagnostics['test']['preprocessor_output']['pr'][0] - - reference = { - 'activity': 'CMIP', - 'dataset': 'HadGEM3-GC31-LL', - 'diagnostic': 'test', - 'ensemble': 'r2i1p1f1', - 'exp': 'historical', - 'frequency': '3hr', - 'grid': 'gn', - 'institute': ['MOHC', 'NERC'], - 'long_name': 'Precipitation', - 'mip': '3hr', - 'modeling_realm': ['atmos'], - 'preprocessor': 'default', - 'project': 'CMIP6', - 'short_name': 'pr', - 'standard_name': 'precipitation_flux', - 'timerange': '2000/2001', - 'units': 'kg m-2 s-1', - } - for key in reference: - assert variable[key] == reference[key] - - -def test_simple_cordex_recipe(tmp_path, patched_datafinder, config_user): - """Test simple CORDEX recipe.""" - content = dedent(""" - diagnostics: - test: - additional_datasets: - - dataset: MOHC-HadGEM3-RA - project: CORDEX - product: output - domain: AFR-44 - institute: MOHC - driver: ECMWF-ERAINT - exp: evaluation - ensemble: r1i1p1 - rcm_version: v1 - start_year: 1991 - end_year: 1993 - mip: mon - variables: - tas: - scripts: null - """) - - recipe = get_recipe(tmp_path, content, config_user) - variable = recipe.diagnostics['test']['preprocessor_output']['tas'][0] - filename = variable.pop('filename').name - assert (filename == - 'CORDEX_MOHC-HadGEM3-RA_v1_ECMWF-ERAINT_AFR-44_mon_evaluation_' - 'r1i1p1_tas_1991-1993.nc') - reference = { - 'alias': 'MOHC-HadGEM3-RA', - 'dataset': 'MOHC-HadGEM3-RA', - 'diagnostic': 'test', - 'domain': 'AFR-44', - 'driver': 'ECMWF-ERAINT', - 'end_year': 1993, - 'ensemble': 'r1i1p1', - 'exp': 'evaluation', - 'frequency': 'mon', - 'institute': 'MOHC', - 'long_name': 'Near-Surface Air Temperature', - 'mip': 'mon', - 'modeling_realm': ['atmos'], - 'preprocessor': 'default', - 'product': 'output', - 'project': 'CORDEX', - 'recipe_dataset_index': 0, - 'rcm_version': 'v1', - 'short_name': 'tas', - 'original_short_name': 'tas', - 'standard_name': 'air_temperature', - 'start_year': 1991, - 'timerange': '1991/1993', - 'units': 'K', - 'variable_group': 'tas', - } - - assert set(variable) == set(reference) - for key in reference: - assert variable[key] == reference[key] - - TEST_ISO_TIMERANGE = [ ('*', '1990-2019'), ('1990/1992', '1990-1992'), @@ -940,7 +635,7 @@ def test_simple_cordex_recipe(tmp_path, patched_datafinder, config_user): @pytest.mark.parametrize('input_time,output_time', TEST_ISO_TIMERANGE) -def test_recipe_iso_timerange(tmp_path, patched_datafinder, config_user, +def test_recipe_iso_timerange(tmp_path, patched_datafinder, session, input_time, output_time): """Test recipe with timerange tag.""" content = dedent(f""" @@ -961,7 +656,7 @@ def test_recipe_iso_timerange(tmp_path, patched_datafinder, config_user, scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) assert len(recipe.tasks) == 2 pr_task = [t for t in recipe.tasks if t.name.endswith('pr')][0] assert len(pr_task.products) == 1 @@ -981,8 +676,8 @@ def test_recipe_iso_timerange(tmp_path, patched_datafinder, config_user, @pytest.mark.parametrize('input_time,output_time', TEST_ISO_TIMERANGE) -def test_recipe_iso_timerange_as_dataset(tmp_path, patched_datafinder, - config_user, input_time, output_time): +def test_recipe_iso_timerange_as_dataset(tmp_path, patched_datafinder, session, + input_time, output_time): """Test recipe with timerange tag in the datasets section.""" content = dedent(f""" datasets: @@ -997,88 +692,31 @@ def test_recipe_iso_timerange_as_dataset(tmp_path, patched_datafinder, variables: pr: mip: 3hr - areacella: - mip: fx + supplementary_variables: + - short_name: areacella + mip: fx scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) - variable = recipe.diagnostics['test']['preprocessor_output']['pr'][0] - filename = variable.pop('filename').name - assert (filename == 'CMIP6_HadGEM3-GC31-LL_3hr_historical_r2i1p1f1_' - f'pr_gn_{output_time}.nc') - fx_variable = ( - recipe.diagnostics['test']['preprocessor_output']['areacella'][0]) - fx_filename = fx_variable.pop('filename').name - assert (fx_filename == - 'CMIP6_HadGEM3-GC31-LL_fx_historical_r2i1p1f1_areacella_gn.nc') - - -TEST_YEAR_FORMAT = [ - ('1/301', '0001/0301'), - ('10/P2Y', '0010/P2Y'), - ('P2Y/10', 'P2Y/0010'), -] - + recipe = get_recipe(tmp_path, content, session) -@pytest.mark.parametrize('input_time,output_time', TEST_YEAR_FORMAT) -def test_update_timerange_year_format(config_user, input_time, output_time): - variable = { - 'project': 'CMIP6', - 'mip': 'Amon', - 'short_name': 'tas', - 'original_short_name': 'tas', - 'dataset': 'HadGEM3-GC31-LL', - 'exp': 'historical', - 'ensemble': 'r2i1p1f1', - 'grid': 'gr', - 'timerange': input_time - } - esmvalcore._recipe.recipe._update_timerange(variable, config_user) - assert variable['timerange'] == output_time - - -def test_update_timerange_no_files_online(config_user): - variable = { - 'alias': 'CMIP6', - 'project': 'CMIP6', - 'mip': 'Amon', - 'short_name': 'tas', - 'original_short_name': 'tas', - 'dataset': 'HadGEM3-GC31-LL', - 'exp': 'historical', - 'ensemble': 'r2i1p1f1', - 'grid': 'gr', - 'timerange': '*/2000', - } - msg = "Missing data for CMIP6: tas. Cannot determine indeterminate time " - with pytest.raises(InputFilesNotFound, match=msg): - esmvalcore._recipe.recipe._update_timerange(variable, config_user) - - -def test_update_timerange_no_files_offline(config_user): - variable = { - 'alias': 'CMIP6', - 'project': 'CMIP6', - 'mip': 'Amon', - 'short_name': 'tas', - 'original_short_name': 'tas', - 'dataset': 'HadGEM3-GC31-LL', - 'exp': 'historical', - 'ensemble': 'r2i1p1f1', - 'grid': 'gr', - 'timerange': '*/2000', - } - config_user = dict(config_user) - config_user['offline'] = False - msg = "Missing data for CMIP6: tas. Cannot determine indeterminate time " - with pytest.raises(InputFilesNotFound, match=msg): - esmvalcore._recipe.recipe._update_timerange(variable, config_user) + assert len(recipe.tasks) == 1 + task = recipe.tasks.pop() + assert len(task.products) == 1 + product = task.products.pop() + filename = ('CMIP6_HadGEM3-GC31-LL_3hr_historical_r2i1p1f1_' + f'pr_gn_{output_time}.nc') + assert product.filename.name == filename + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert len(dataset.supplementaries) == 1 + supplementary_ds = dataset.supplementaries[0] + assert supplementary_ds.facets['short_name'] == 'areacella' + assert 'timerange' not in supplementary_ds.facets -def test_reference_dataset(tmp_path, patched_datafinder, config_user, - monkeypatch): +def test_reference_dataset(tmp_path, patched_datafinder, session, monkeypatch): levels = [100] get_reference_levels = create_autospec( esmvalcore._recipe.recipe.get_reference_levels, return_value=levels) @@ -1113,19 +751,18 @@ def test_reference_dataset(tmp_path, patched_datafinder, config_user, end_year: 2005 ensemble: r1i1p1 additional_datasets: - - {dataset: GFDL-CM3} - - {dataset: MPI-ESM-LR} + - dataset: GFDL-CM3 + - dataset: MPI-ESM-LR reference_dataset: MPI-ESM-LR ch4: <<: *var preprocessor: test_from_cmor_table additional_datasets: - - {dataset: GFDL-CM3} + - dataset: GFDL-CM3 scripts: null """) - - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) assert len(recipe.tasks) == 2 @@ -1138,19 +775,10 @@ def test_reference_dataset(tmp_path, patched_datafinder, config_user, reference = next(p for p in task.products if p.attributes['dataset'] == 'MPI-ESM-LR') - assert product.settings['regrid']['target_grid'] == reference.files[0] + assert product.settings['regrid']['target_grid'] == reference.datasets[0] assert product.settings['extract_levels']['levels'] == levels - fix_dir = os.path.splitext(reference.filename)[0] + '_fixed' - get_reference_levels.assert_called_once_with( - filename=reference.files[0], - project='CMIP5', - dataset='MPI-ESM-LR', - short_name='ta', - mip='Amon', - frequency='mon', - fix_dir=fix_dir, - ) + get_reference_levels.assert_called_once_with(reference.datasets[0]) assert 'regrid' not in reference.settings assert 'extract_levels' not in reference.settings @@ -1180,8 +808,39 @@ def test_reference_dataset(tmp_path, patched_datafinder, config_user, ] -def test_custom_preproc_order(tmp_path, patched_datafinder, config_user): +def test_reference_dataset_undefined(tmp_path, monkeypatch, session): + content = dedent(""" + preprocessors: + test_from_reference: + extract_levels: + levels: reference_dataset + scheme: linear + + diagnostics: + diagnostic_name: + variables: + ta: &var + preprocessor: test_from_reference + project: CMIP5 + mip: Amon + exp: historical + start_year: 2000 + end_year: 2005 + ensemble: r1i1p1 + additional_datasets: + - dataset: GFDL-CM3 + - dataset: MPI-ESM-LR + + scripts: null + """) + with pytest.raises(RecipeError) as error: + get_recipe(tmp_path, content, session) + msg = ("Preprocessor 'test_from_reference' uses 'reference_dataset', but " + "'reference_dataset' is not defined") + assert msg in error.value.failed_tasks[0].message + +def test_custom_preproc_order(tmp_path, patched_datafinder, session): content = dedent(""" preprocessors: default: &default @@ -1231,7 +890,7 @@ def test_custom_preproc_order(tmp_path, patched_datafinder, config_user): scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) assert len(recipe.tasks) == 4 @@ -1260,15 +919,11 @@ def test_custom_preproc_order(tmp_path, patched_datafinder, config_user): 'end_month': 6, 'end_day': 28, } - assert product.settings['clip_timerange'] == { - 'timerange': '2000/2005', - } else: assert False, f"invalid task {task.name}" -def test_derive(tmp_path, patched_datafinder, config_user): - +def test_derive(tmp_path, patched_datafinder, session): content = dedent(""" diagnostics: diagnostic_name: @@ -1286,38 +941,25 @@ def test_derive(tmp_path, patched_datafinder, config_user): scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'toz' - assert len(task.ancestors) == 2 - assert 'diagnostic_name' + TASKSEP + 'toz_derive_input_ps' in [ - t.name for t in task.ancestors - ] - assert 'diagnostic_name' + TASKSEP + 'toz_derive_input_tro3' in [ - t.name for t in task.ancestors - ] # Check product content of tasks assert len(task.products) == 1 product = task.products.pop() assert 'derive' in product.settings assert product.attributes['short_name'] == 'toz' - assert product.files - - ps_product = next(p for a in task.ancestors for p in a.products - if p.attributes['short_name'] == 'ps') - tro3_product = next(p for a in task.ancestors for p in a.products - if p.attributes['short_name'] == 'tro3') - assert ps_product.filename in product.files - assert tro3_product.filename in product.files + assert len(product.datasets) == 2 + input_variables = {d.facets['short_name'] for d in product.datasets} + assert input_variables == {'ps', 'tro3'} -def test_derive_not_needed(tmp_path, patched_datafinder, config_user): +def test_derive_not_needed(tmp_path, patched_datafinder, session): content = dedent(""" diagnostics: diagnostic_name: @@ -1335,37 +977,26 @@ def test_derive_not_needed(tmp_path, patched_datafinder, config_user): scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 task = recipe.tasks.pop() - assert task.name == 'diagnostic_name/toz' - assert len(task.ancestors) == 1 - ancestor = [t for t in task.ancestors][0] - assert ancestor.name == 'diagnostic_name/toz_derive_input_toz' # Check product content of tasks assert len(task.products) == 1 product = task.products.pop() - assert product.attributes['short_name'] == 'toz' - assert 'derive' in product.settings - - assert len(ancestor.products) == 1 - ancestor_product = ancestor.products.pop() - assert ancestor_product.filename in product.files - assert ancestor_product.attributes['short_name'] == 'toz' - assert 'derive' not in ancestor_product.settings + assert 'derive' not in product.settings - # Check that fixes are applied just once - fixes = ('fix_file', 'fix_metadata', 'fix_data') - for fix in fixes: - assert fix in ancestor_product.settings - assert fix not in product.settings + # Check dataset + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert dataset.facets['short_name'] == 'toz' + assert dataset.files -def test_derive_with_fx_ohc(tmp_path, patched_datafinder, config_user): +def test_derive_with_fx_ohc(tmp_path, patched_datafinder, session): content = dedent(""" diagnostics: diagnostic_name: @@ -1386,7 +1017,7 @@ def test_derive_with_fx_ohc(tmp_path, patched_datafinder, config_user): scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -1394,33 +1025,27 @@ def test_derive_with_fx_ohc(tmp_path, patched_datafinder, config_user): assert task.name == 'diagnostic_name' + TASKSEP + 'ohc' # Check products - all_product_files = [] assert len(task.products) == 3 for product in task.products: assert 'derive' in product.settings assert product.attributes['short_name'] == 'ohc' - all_product_files.extend(product.files) - - # Check ancestors - assert len(task.ancestors) == 2 - assert task.ancestors[0].name == ( - 'diagnostic_name/ohc_derive_input_thetao') - assert task.ancestors[1].name == ( - 'diagnostic_name/ohc_derive_input_volcello') - for ancestor_product in task.ancestors[0].products: - assert ancestor_product.attributes['short_name'] == 'thetao' - assert ancestor_product.filename in all_product_files - for ancestor_product in task.ancestors[1].products: - assert ancestor_product.attributes['short_name'] == 'volcello' - if ancestor_product.attributes['project'] == 'CMIP6': - assert ancestor_product.attributes['mip'] == 'Ofx' + + # Check datasets + assert len(product.datasets) == 2 + thetao_ds = next(d for d in product.datasets + if d.facets['short_name'] == 'thetao') + assert thetao_ds.facets['mip'] == 'Omon' + volcello_ds = next(d for d in product.datasets + if d.facets['short_name'] == 'volcello') + if volcello_ds.facets['project'] == 'CMIP6': + mip = 'Ofx' else: - assert ancestor_product.attributes['mip'] == 'fx' - assert ancestor_product.filename in all_product_files + mip = 'fx' + assert volcello_ds.facets['mip'] == mip def test_derive_with_fx_ohc_fail(tmp_path, patched_failing_datafinder, - config_user): + session): content = dedent(""" diagnostics: diagnostic_name: @@ -1442,11 +1067,11 @@ def test_derive_with_fx_ohc_fail(tmp_path, patched_failing_datafinder, scripts: null """) with pytest.raises(RecipeError): - get_recipe(tmp_path, content, config_user) + get_recipe(tmp_path, content, session) def test_derive_with_optional_var(tmp_path, patched_datafinder, - patched_tas_derivation, config_user): + patched_tas_derivation, session): content = dedent(""" diagnostics: diagnostic_name: @@ -1467,7 +1092,7 @@ def test_derive_with_optional_var(tmp_path, patched_datafinder, scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -1475,28 +1100,23 @@ def test_derive_with_optional_var(tmp_path, patched_datafinder, assert task.name == 'diagnostic_name' + TASKSEP + 'tas' # Check products - all_product_files = [] assert len(task.products) == 3 for product in task.products: assert 'derive' in product.settings assert product.attributes['short_name'] == 'tas' - all_product_files.extend(product.files) - - # Check ancestors - assert len(task.ancestors) == 2 - assert task.ancestors[0].name == ('diagnostic_name/tas_derive_input_pr') - assert task.ancestors[1].name == ( - 'diagnostic_name/tas_derive_input_areacella') - for ancestor_product in task.ancestors[0].products: - assert ancestor_product.attributes['short_name'] == 'pr' - assert ancestor_product.filename in all_product_files - for ancestor_product in task.ancestors[1].products: - assert ancestor_product.attributes['short_name'] == 'areacella' - assert ancestor_product.filename in all_product_files + assert len(product.datasets) == 2 + pr_ds = next(d for d in product.datasets + if d.facets['short_name'] == 'pr') + assert pr_ds.facets['mip'] == 'Amon' + assert pr_ds.facets['timerange'] == '2000/2005' + areacella_ds = next(d for d in product.datasets + if d.facets['short_name'] == 'areacella') + assert areacella_ds.facets['mip'] == 'fx' + assert 'timerange' not in areacella_ds.facets def test_derive_with_optional_var_nodata(tmp_path, patched_failing_datafinder, - patched_tas_derivation, config_user): + patched_tas_derivation, session): content = dedent(""" diagnostics: diagnostic_name: @@ -1517,7 +1137,7 @@ def test_derive_with_optional_var_nodata(tmp_path, patched_failing_datafinder, scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -1525,24 +1145,17 @@ def test_derive_with_optional_var_nodata(tmp_path, patched_failing_datafinder, assert task.name == 'diagnostic_name' + TASKSEP + 'tas' # Check products - all_product_files = [] assert len(task.products) == 3 for product in task.products: assert 'derive' in product.settings assert product.attributes['short_name'] == 'tas' - all_product_files.extend(product.files) - - # Check ancestors - assert len(task.ancestors) == 1 - assert task.ancestors[0].name == ('diagnostic_name/tas_derive_input_pr') - for ancestor_product in task.ancestors[0].products: - assert ancestor_product.attributes['short_name'] == 'pr' - assert ancestor_product.filename in all_product_files + # Check datasets + assert len(product.datasets) == 1 + assert product.datasets[0].facets['short_name'] == 'pr' -def test_derive_contains_start_end_year(tmp_path, patched_datafinder, - config_user): +def test_derive_contains_start_end_year(tmp_path, patched_datafinder, session): content = dedent(""" diagnostics: diagnostic_name: @@ -1559,7 +1172,7 @@ def test_derive_contains_start_end_year(tmp_path, patched_datafinder, scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -1575,10 +1188,11 @@ def test_derive_contains_start_end_year(tmp_path, patched_datafinder, assert product.attributes['end_year'] == 2005 -def test_derive_timerange_wildcard(tmp_path, patched_datafinder, - config_user): +@pytest.mark.parametrize('force_derivation', [True, False]) +def test_derive_timerange_wildcard(tmp_path, patched_datafinder, session, + force_derivation): - content = dedent(""" + content = dedent(f""" diagnostics: diagnostic_name: variables: @@ -1588,13 +1202,14 @@ def test_derive_timerange_wildcard(tmp_path, patched_datafinder, exp: historical timerange: '*' derive: true - force_derivation: true + force_derivation: {force_derivation} additional_datasets: - - {dataset: GFDL-CM3, ensemble: r1i1p1} + - dataset: GFDL-CM3 + ensemble: r1i1p1 scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -1603,44 +1218,14 @@ def test_derive_timerange_wildcard(tmp_path, patched_datafinder, # Check that start_year and end_year are present in attributes assert len(task.products) == 1 product = task.products.pop() - assert 'derive' in product.settings + if force_derivation: + assert 'derive' in product.settings assert product.attributes['short_name'] == 'toz' assert product.attributes['timerange'] == '1990/2019' assert product.attributes['start_year'] == 1990 assert product.attributes['end_year'] == 2019 -def test_derive_fail_timerange_wildcard(tmp_path, patched_datafinder, - config_user): - - content = dedent(""" - diagnostics: - diagnostic_name: - variables: - toz: - project: CMIP5 - mip: Amon - exp: historical - timerange: '*' - derive: true - force_derivation: false - additional_datasets: - - {dataset: GFDL-CM3, ensemble: r1i1p1} - scripts: null - """) - msg = ( - "Error in derived variable: toz: " - "Using 'force_derivation: false' (the default option) " - "in combination with wildcards ('*') in timerange is " - "not allowed; explicitly use 'force_derivation: true' " - "or avoid the use of wildcards in timerange") - - with pytest.raises(RecipeError) as rec_err: - get_recipe(tmp_path, content, config_user) - - assert msg in rec_err.value.failed_tasks[0].message - - def create_test_image(basename, cfg): """Get a valid path for saving a diagnostic plot.""" image = Path(cfg['plot_dir']) / (basename + '.' + cfg['output_file_type']) @@ -1657,6 +1242,14 @@ def get_diagnostic_filename(basename, cfg, extension='nc'): ) +def simulate_preprocessor_run(task): + """Simulate preprocessor run.""" + task._initialize_product_provenance() + for product in task.products: + create_test_file(product.filename) + product.save_provenance() + + def simulate_diagnostic_run(diagnostic_task): """Simulate Python diagnostic run.""" cfg = diagnostic_task.settings @@ -1688,7 +1281,7 @@ def simulate_diagnostic_run(diagnostic_task): def test_diagnostic_task_provenance( tmp_path, patched_datafinder, - config_user, + session, ): script = tmp_path / 'diagnostic.py' script.write_text('') @@ -1720,7 +1313,11 @@ def test_diagnostic_task_provenance( ancestors: [script_name] """.format(script=script)) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) + preproc_task = next(t for t in recipe.tasks.flatten() + if isinstance(t, PreprocessingTask)) + simulate_preprocessor_run(preproc_task) + diagnostic_task = recipe.tasks.pop() simulate_diagnostic_run(next(iter(diagnostic_task.ancestors))) @@ -1757,7 +1354,7 @@ def test_diagnostic_task_provenance( assert recipe_record[0].get_attribute('attribute:' + key).pop() == value - # Test that provenance was saved to netcdf, xml and svg plot + # Test that provenance was saved to xml and info embedded in netcdf product = next( iter(p for p in diagnostic_task.products if p.filename.endswith('.nc'))) @@ -1768,7 +1365,7 @@ def test_diagnostic_task_provenance( assert os.path.exists(prefix + '.xml') -def test_alias_generation(tmp_path, patched_datafinder, config_user): +def test_alias_generation(tmp_path, patched_datafinder, session): content = dedent(""" diagnostics: diagnostic_name: @@ -1794,20 +1391,18 @@ def test_alias_generation(tmp_path, patched_datafinder, config_user): - {dataset: FGOALS-g3, sub_experiment: s1961, ensemble: r1} - {project: OBS, dataset: ERA-Interim, version: 1} - {project: OBS, dataset: ERA-Interim, version: 2} - - {project: CMIP6, activity: CMP, dataset: GF3, ensemble: r1} - - {project: CMIP6, activity: CMP, dataset: GF2, ensemble: r1} - - {project: CMIP6, activity: HRMP, dataset: EC, ensemble: r1} - - {project: CMIP6, activity: HRMP, dataset: HA, ensemble: r1} + - {project: CMIP6, activity: CMP, dataset: GF3, ensemble: r1, institute: fake} + - {project: CMIP6, activity: CMP, dataset: GF2, ensemble: r1, institute: fake} + - {project: CMIP6, activity: HRMP, dataset: EC, ensemble: r1, institute: fake} + - {project: CMIP6, activity: HRMP, dataset: HA, ensemble: r1, institute: fake} - {project: CORDEX, driver: ICHEC-EC-EARTH, dataset: SMHI-RCA4, ensemble: r1, mip: mon} - {project: CORDEX, driver: MIROC-MIROC5, dataset: SMHI-RCA4, ensemble: r1, mip: mon} scripts: null """) # noqa: - recipe = get_recipe(tmp_path, content, config_user) - assert len(recipe.diagnostics) == 1 - diag = recipe.diagnostics['diagnostic_name'] - var = diag['preprocessor_output']['pr'] - for dataset in var: + recipe = get_recipe(tmp_path, content, session) + assert len(recipe.datasets) == 14 + for dataset in recipe.datasets: if dataset['project'] == 'CMIP5': if dataset['dataset'] == 'GFDL-CM3': assert dataset['alias'] == 'CMIP5_GFDL-CM3' @@ -1844,7 +1439,7 @@ def test_alias_generation(tmp_path, patched_datafinder, config_user): assert dataset['alias'] == 'OBS_2' -def test_concatenation(tmp_path, patched_datafinder, config_user): +def test_concatenation(tmp_path, patched_datafinder, session): content = dedent(""" diagnostics: diagnostic_name: @@ -1868,18 +1463,16 @@ def test_concatenation(tmp_path, patched_datafinder, config_user): scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) - assert len(recipe.diagnostics) == 1 - diag = recipe.diagnostics['diagnostic_name'] - var = diag['preprocessor_output']['ta'] - for dataset in var: + recipe = get_recipe(tmp_path, content, session) + assert len(recipe.datasets) == 2 + for dataset in recipe.datasets: if dataset['exp'] == 'historical': assert dataset['alias'] == 'historical' else: assert dataset['alias'] == 'historical-rcp85' -def test_ensemble_expansion(tmp_path, patched_datafinder, config_user): +def test_ensemble_expansion(tmp_path, patched_datafinder, session): content = dedent(""" diagnostics: diagnostic_name: @@ -1900,17 +1493,14 @@ def test_ensemble_expansion(tmp_path, patched_datafinder, config_user): scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) - assert len(recipe.diagnostics) == 1 - diag = recipe.diagnostics['diagnostic_name'] - var = diag['preprocessor_output']['ta'] - assert len(var) == 3 - assert var[0]['ensemble'] == 'r1i1p1' - assert var[1]['ensemble'] == 'r2i1p1' - assert var[2]['ensemble'] == 'r3i1p1' + recipe = get_recipe(tmp_path, content, session) + assert len(recipe.datasets) == 3 + assert recipe.datasets[0]['ensemble'] == 'r1i1p1' + assert recipe.datasets[1]['ensemble'] == 'r2i1p1' + assert recipe.datasets[2]['ensemble'] == 'r3i1p1' -def test_extract_shape(tmp_path, patched_datafinder, config_user): +def test_extract_shape(tmp_path, patched_datafinder, session): TAGS.set_tag_values(TAGS_FOR_TESTING) content = dedent(""" @@ -1935,11 +1525,11 @@ def test_extract_shape(tmp_path, patched_datafinder, config_user): scripts: null """) # Create shapefile - shapefile = config_user['auxiliary_data_dir'] / Path('test.shp') + shapefile = session['auxiliary_data_dir'] / Path('test.shp') shapefile.parent.mkdir(parents=True, exist_ok=True) shapefile.touch() - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) assert len(recipe.tasks) == 1 task = recipe.tasks.pop() @@ -1950,12 +1540,12 @@ def test_extract_shape(tmp_path, patched_datafinder, config_user): @pytest.mark.parametrize('invalid_arg', ['shapefile', 'method', 'crop', 'decomposed']) -def test_extract_shape_raises(tmp_path, patched_datafinder, config_user, +def test_extract_shape_raises(tmp_path, patched_datafinder, session, invalid_arg): TAGS.set_tag_values(TAGS_FOR_TESTING) # Create shapefile - shapefile = config_user['auxiliary_data_dir'] / Path('test.shp') + shapefile = session['auxiliary_data_dir'] / Path('test.shp') shapefile.parent.mkdir(parents=True, exist_ok=True) shapefile.touch() @@ -1990,7 +1580,7 @@ def test_extract_shape_raises(tmp_path, patched_datafinder, config_user, content = yaml.safe_dump(recipe) with pytest.raises(RecipeError) as exc: - get_recipe(tmp_path, content, config_user) + get_recipe(tmp_path, content, session) assert str(exc.value) == INITIALIZATION_ERROR_MSG assert 'extract_shape' in exc.value.failed_tasks[0].message @@ -2017,7 +1607,7 @@ def _test_output_product_consistency(products, preprocessor, statistics): return product_out -def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): +def test_ensemble_statistics(tmp_path, patched_datafinder, session): statistics = ['mean', 'max'] diagnostic = 'diagnostic_name' variable = 'pr' @@ -2049,9 +1639,8 @@ def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) - variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] - datasets = set([var['dataset'] for var in variable]) + recipe = get_recipe(tmp_path, content, session) + datasets = set([ds['dataset'] for ds in recipe.datasets]) task = next(iter(recipe.tasks)) products = task.products @@ -2064,7 +1653,7 @@ def test_ensemble_statistics(tmp_path, patched_datafinder, config_user): assert next(iter(products)).provenance is not None -def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): +def test_multi_model_statistics(tmp_path, patched_datafinder, session): statistics = ['mean', 'max'] diagnostic = 'diagnostic_name' variable = 'pr' @@ -2097,8 +1686,7 @@ def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) - variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] + recipe = get_recipe(tmp_path, content, session) task = next(iter(recipe.tasks)) products = task.products @@ -2111,9 +1699,7 @@ def test_multi_model_statistics(tmp_path, patched_datafinder, config_user): assert next(iter(products)).provenance is not None -def test_multi_model_statistics_exclude(tmp_path, - patched_datafinder, - config_user): +def test_multi_model_statistics_exclude(tmp_path, patched_datafinder, session): statistics = ['mean', 'max'] diagnostic = 'diagnostic_name' variable = 'pr' @@ -2150,8 +1736,7 @@ def test_multi_model_statistics_exclude(tmp_path, scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) - variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] + recipe = get_recipe(tmp_path, content, session) task = next(iter(recipe.tasks)) products = task.products @@ -2167,8 +1752,7 @@ def test_multi_model_statistics_exclude(tmp_path, assert next(iter(products)).provenance is not None -def test_groupby_combined_statistics(tmp_path, patched_datafinder, - config_user): +def test_groupby_combined_statistics(tmp_path, patched_datafinder, session): diagnostic = 'diagnostic_name' variable = 'pr' @@ -2210,9 +1794,8 @@ def test_groupby_combined_statistics(tmp_path, patched_datafinder, scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) - variable = recipe.diagnostics[diagnostic]['preprocessor_output'][variable] - datasets = set([var['dataset'] for var in variable]) + recipe = get_recipe(tmp_path, content, session) + datasets = set([ds['dataset'] for ds in recipe.datasets]) products = next(iter(recipe.tasks)).products @@ -2233,7 +1816,7 @@ def test_groupby_combined_statistics(tmp_path, patched_datafinder, mm_products) == len(mm_statistics) * len(ens_statistics) * len(groupby) -def test_weighting_landsea_fraction(tmp_path, patched_datafinder, config_user): +def test_weighting_landsea_fraction(tmp_path, patched_datafinder, session): TAGS.set_tag_values(TAGS_FOR_TESTING) content = dedent(""" @@ -2259,7 +1842,7 @@ def test_weighting_landsea_fraction(tmp_path, patched_datafinder, config_user): tier: 1} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -2273,19 +1856,22 @@ def test_weighting_landsea_fraction(tmp_path, patched_datafinder, config_user): settings = product.settings['weighting_landsea_fraction'] assert len(settings) == 1 assert settings['area_type'] == 'land' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - if product.attributes['project'] == 'obs4MIPs': - assert len(fx_variables) == 1 - assert fx_variables.get('sftlf') + assert len(product.datasets) == 1 + dataset = product.datasets[0] + short_names = { + ds.facets['short_name'] + for ds in dataset.supplementaries + } + if dataset.facets['project'] == 'obs4MIPs': + assert len(dataset.supplementaries) == 1 + assert {'sftlf'} == short_names else: - assert len(fx_variables) == 2 - assert fx_variables.get('sftlf') - assert fx_variables.get('sftof') + assert len(dataset.supplementaries) == 2 + assert {'sftlf', 'sftof'} == short_names def test_weighting_landsea_fraction_no_fx(tmp_path, patched_failing_datafinder, - config_user): + session): content = dedent(""" preprocessors: landfrac_weighting: @@ -2310,28 +1896,13 @@ def test_weighting_landsea_fraction_no_fx(tmp_path, patched_failing_datafinder, tier: 1} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) - # Check generated tasks - assert len(recipe.tasks) == 1 - task = recipe.tasks.pop() - assert task.name == 'diagnostic_name' + TASKSEP + 'gpp' - - # Check weighting - assert len(task.products) == 2 - for product in task.products: - assert 'weighting_landsea_fraction' in product.settings - settings = product.settings['weighting_landsea_fraction'] - assert len(settings) == 1 - assert 'exclude' not in settings - assert settings['area_type'] == 'land' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 0 + with pytest.raises(RecipeError): + get_recipe(tmp_path, content, session) def test_weighting_landsea_fraction_exclude(tmp_path, patched_datafinder, - config_user): + session): content = dedent(""" preprocessors: landfrac_weighting: @@ -2358,7 +1929,7 @@ def test_weighting_landsea_fraction_exclude(tmp_path, patched_datafinder, tier: 1} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -2379,7 +1950,7 @@ def test_weighting_landsea_fraction_exclude(tmp_path, patched_datafinder, def test_weighting_landsea_fraction_exclude_fail(tmp_path, patched_datafinder, - config_user): + session): content = dedent(""" preprocessors: landfrac_weighting: @@ -2405,15 +1976,15 @@ def test_weighting_landsea_fraction_exclude_fail(tmp_path, patched_datafinder, scripts: null """) with pytest.raises(RecipeError) as exc_info: - get_recipe(tmp_path, content, config_user) + get_recipe(tmp_path, content, session) assert str(exc_info.value) == INITIALIZATION_ERROR_MSG assert str(exc_info.value.failed_tasks[0].message) == ( - 'Preprocessor landfrac_weighting uses alternative_dataset, but ' - 'alternative_dataset is not defined for variable gpp of diagnostic ' - 'diagnostic_name') + "Preprocessor 'landfrac_weighting' uses 'alternative_dataset', but " + "'alternative_dataset' is not defined for variable 'gpp' of " + "diagnostic 'diagnostic_name'.") -def test_area_statistics(tmp_path, patched_datafinder, config_user): +def test_area_statistics(tmp_path, patched_datafinder, session): content = dedent(""" preprocessors: area_statistics: @@ -2437,7 +2008,7 @@ def test_area_statistics(tmp_path, patched_datafinder, config_user): tier: 1} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -2451,18 +2022,19 @@ def test_area_statistics(tmp_path, patched_datafinder, config_user): settings = product.settings['area_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - if product.attributes['project'] == 'obs4MIPs': - assert len(fx_variables) == 1 - assert fx_variables.get('areacella') + assert len(product.datasets) == 1 + dataset = product.datasets[0] + short_names = { + ds.facets['short_name'] + for ds in dataset.supplementaries + } + if dataset.facets['project'] == 'obs4MIPs': + assert short_names == {'areacella'} else: - assert len(fx_variables) == 2 - assert fx_variables.get('areacella') - assert fx_variables.get('areacello') + assert short_names == {'areacella', 'areacello'} -def test_landmask(tmp_path, patched_datafinder, config_user): +def test_landmask(tmp_path, patched_datafinder, session): content = dedent(""" preprocessors: landmask: @@ -2486,7 +2058,7 @@ def test_landmask(tmp_path, patched_datafinder, config_user): tier: 1} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -2500,16 +2072,15 @@ def test_landmask(tmp_path, patched_datafinder, config_user): settings = product.settings['mask_landsea'] assert len(settings) == 1 assert settings['mask_out'] == 'sea' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - fx_variables = fx_variables.values() - if product.attributes['project'] == 'obs4MIPs': - assert len(fx_variables) == 1 + assert len(product.datasets) == 1 + dataset = product.datasets[0] + if dataset.facets['project'] == 'obs4MIPs': + assert len(dataset.supplementaries) == 1 else: - assert len(fx_variables) == 2 + assert len(dataset.supplementaries) == 2 -def test_empty_fxvar_none(tmp_path, patched_datafinder, config_user): +def test_empty_fxvar_none(tmp_path, patched_datafinder, session): """Test that no fx variables are added if explicitly specified.""" content = dedent(""" preprocessors: @@ -2532,15 +2103,16 @@ def test_empty_fxvar_none(tmp_path, patched_datafinder, config_user): - {dataset: CanESM2} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check that no custom fx variables are present task = recipe.tasks.pop() product = task.products.pop() - assert product.settings['add_fx_variables']['fx_variables'] == {} + dataset = product.datasets[0] + assert dataset.supplementaries == [] -def test_empty_fxvar_list(tmp_path, patched_datafinder, config_user): +def test_empty_fxvar_list(tmp_path, patched_datafinder, session): """Test that no fx variables are added if explicitly specified.""" content = dedent(""" preprocessors: @@ -2563,15 +2135,16 @@ def test_empty_fxvar_list(tmp_path, patched_datafinder, config_user): - {dataset: CanESM2} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check that no custom fx variables are present task = recipe.tasks.pop() product = task.products.pop() - assert product.settings['add_fx_variables']['fx_variables'] == {} + dataset = product.datasets[0] + assert dataset.supplementaries == [] -def test_empty_fxvar_dict(tmp_path, patched_datafinder, config_user): +def test_empty_fxvar_dict(tmp_path, patched_datafinder, session): """Test that no fx variables are added if explicitly specified.""" content = dedent(""" preprocessors: @@ -2594,12 +2167,14 @@ def test_empty_fxvar_dict(tmp_path, patched_datafinder, config_user): - {dataset: CanESM2} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check that no custom fx variables are present task = recipe.tasks.pop() product = task.products.pop() - assert product.settings['add_fx_variables']['fx_variables'] == {} + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert dataset.supplementaries == [] @pytest.mark.parametrize('content', [ @@ -2672,43 +2247,8 @@ def test_empty_fxvar_dict(tmp_path, patched_datafinder, config_user): """), id='fx_variables_as_list_of_dicts'), ]) -def test_user_defined_fxvar( - tmp_path, - patched_datafinder, - config_user, - content, -): - content = dedent(""" - preprocessors: - landmask: - mask_landsea: - mask_out: sea - fx_variables: [{'short_name': 'sftlf', 'exp': 'piControl'}] - mask_landseaice: - mask_out: sea - fx_variables: [{'short_name': 'sftgif', 'exp': 'piControl'}] - volume_statistics: - operator: mean - area_statistics: - operator: mean - fx_variables: [{'short_name': 'areacello', 'mip': 'fx', - 'exp': 'piControl'}] - diagnostics: - diagnostic_name: - variables: - gpp: - preprocessor: landmask - project: CMIP5 - mip: Lmon - exp: historical - start_year: 2000 - end_year: 2005 - ensemble: r1i1p1 - additional_datasets: - - {dataset: CanESM2} - scripts: null - """) - recipe = get_recipe(tmp_path, content, config_user) +def test_user_defined_fxvar(tmp_path, patched_datafinder, session, content): + recipe = get_recipe(tmp_path, content, session) # Check custom fx variables task = recipe.tasks.pop() @@ -2718,34 +2258,42 @@ def test_user_defined_fxvar( settings = product.settings['mask_landsea'] assert len(settings) == 1 assert settings['mask_out'] == 'sea' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 4 - assert '_fx_' in fx_variables['sftlf']['filename'].name - assert '_piControl_' in fx_variables['sftlf']['filename'].name + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert isinstance(dataset.supplementaries, list) + supplementaries = { + ds.facets['short_name']: ds + for ds in dataset.supplementaries + } + assert len(list(supplementaries)) == 4 + sftlf_ds = supplementaries['sftlf'] + assert sftlf_ds.facets['mip'] == 'fx' + assert sftlf_ds.facets['exp'] == 'piControl' # landseaice settings = product.settings['mask_landseaice'] assert len(settings) == 1 assert settings['mask_out'] == 'sea' - assert '_fx_' in fx_variables['sftlf']['filename'].name - assert '_piControl_' in fx_variables['sftlf']['filename'].name + sftgif_ds = supplementaries['sftgif'] + assert sftgif_ds.facets['mip'] == 'fx' + assert sftgif_ds.facets['exp'] == 'piControl' # volume statistics settings = product.settings['volume_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' - assert 'volcello' in fx_variables + assert 'volcello' in supplementaries # area statistics settings = product.settings['area_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' - assert '_fx_' in fx_variables['areacello']['filename'].name - assert '_piControl_' in fx_variables['areacello']['filename'].name + areacello_ds = supplementaries['areacello'] + assert areacello_ds.facets['mip'] == 'fx' + assert areacello_ds.facets['exp'] == 'piControl' -def test_landmask_no_fx(tmp_path, patched_failing_datafinder, config_user): +def test_landmask_no_fx(tmp_path, patched_failing_datafinder, session): content = dedent(""" preprocessors: landmask: @@ -2771,7 +2319,7 @@ def test_landmask_no_fx(tmp_path, patched_failing_datafinder, config_user): tier: 1} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -2785,13 +2333,12 @@ def test_landmask_no_fx(tmp_path, patched_failing_datafinder, config_user): settings = product.settings['mask_landsea'] assert len(settings) == 1 assert settings['mask_out'] == 'sea' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - fx_variables = fx_variables.values() - assert not any(fx_variables) + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert dataset.supplementaries == [] -def test_fx_vars_fixed_mip_cmip6(tmp_path, patched_datafinder, config_user): +def test_fx_vars_fixed_mip_cmip6(tmp_path, patched_datafinder, session): """Test fx variables with given mips.""" TAGS.set_tag_values(TAGS_FOR_TESTING) @@ -2826,7 +2373,7 @@ def test_fx_vars_fixed_mip_cmip6(tmp_path, patched_datafinder, config_user): - {dataset: CanESM5} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -2841,16 +2388,22 @@ def test_fx_vars_fixed_mip_cmip6(tmp_path, patched_datafinder, config_user): assert len(settings) == 1 assert settings['operator'] == 'mean' - # Check add_fx_variables - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 2 - assert '_fx_' in fx_variables['sftgif']['filename'].name - assert '_r2i1p1f1_' in fx_variables['volcello']['filename'].name - assert '_Ofx_' in fx_variables['volcello']['filename'].name + # Check legacy method of adding supplementary variables + assert len(product.datasets) == 1 + dataset = product.datasets[0] + supplementaries = { + ds.facets['short_name']: ds + for ds in dataset.supplementaries + } + assert len(list(supplementaries)) == 2 + sftgif_ds = supplementaries['sftgif'] + assert sftgif_ds.facets['mip'] == 'fx' + volcello_ds = supplementaries['volcello'] + assert volcello_ds.facets['ensemble'] == 'r2i1p1f1' + assert volcello_ds.facets['mip'] == 'Ofx' -def test_fx_vars_invalid_mip_cmip6(tmp_path, patched_datafinder, config_user): +def test_fx_vars_invalid_mip_cmip6(tmp_path, patched_datafinder, session): """Test fx variables with invalid mip.""" TAGS.set_tag_values(TAGS_FOR_TESTING) @@ -2879,16 +2432,16 @@ def test_fx_vars_invalid_mip_cmip6(tmp_path, patched_datafinder, config_user): - {dataset: CanESM5} scripts: null """) - msg = ("Requested mip table 'INVALID' for fx variable 'areacella' not " - "available for project 'CMIP6'") + msg = ("Unable to load CMOR table (project) 'CMIP6' for variable " + "'areacella' with mip 'INVALID'") with pytest.raises(RecipeError) as rec_err_exp: - get_recipe(tmp_path, content, config_user) + get_recipe(tmp_path, content, session) assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG assert msg in rec_err_exp.value.failed_tasks[0].message def test_fx_vars_invalid_mip_for_var_cmip6(tmp_path, patched_datafinder, - config_user): + session): """Test fx variables with invalid mip for variable.""" TAGS.set_tag_values(TAGS_FOR_TESTING) @@ -2917,15 +2470,15 @@ def test_fx_vars_invalid_mip_for_var_cmip6(tmp_path, patched_datafinder, - {dataset: CanESM5} scripts: null """) - msg = ("fx variable 'areacella' not available in CMOR table 'Lmon' for " - "'CMIP6'") + msg = ("Unable to load CMOR table (project) 'CMIP6' for variable " + "'areacella' with mip 'Lmon'") with pytest.raises(RecipeError) as rec_err_exp: - get_recipe(tmp_path, content, config_user) + get_recipe(tmp_path, content, session) assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG assert msg in rec_err_exp.value.failed_tasks[0].message -def test_fx_vars_mip_search_cmip6(tmp_path, patched_datafinder, config_user): +def test_fx_vars_mip_search_cmip6(tmp_path, patched_datafinder, session): """Test mip tables search for different fx variables.""" TAGS.set_tag_values(TAGS_FOR_TESTING) @@ -2937,7 +2490,6 @@ def test_fx_vars_mip_search_cmip6(tmp_path, patched_datafinder, config_user): fx_variables: areacella: areacello: - clayfrac: mask_landsea: mask_out: sea @@ -2957,7 +2509,7 @@ def test_fx_vars_mip_search_cmip6(tmp_path, patched_datafinder, config_user): - {dataset: CanESM5} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -2978,18 +2530,21 @@ def test_fx_vars_mip_search_cmip6(tmp_path, patched_datafinder, config_user): assert len(settings) == 1 assert settings['mask_out'] == 'sea' - # Check add_fx_variables - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 5 - assert '_fx_' in fx_variables['areacella']['filename'].name - assert '_Ofx_' in fx_variables['areacello']['filename'].name - assert '_Efx_' in fx_variables['clayfrac']['filename'].name - assert '_fx_' in fx_variables['sftlf']['filename'].name - assert '_Ofx_' in fx_variables['sftof']['filename'].name + # Check legacy method of adding supplementary variables + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert len(dataset.supplementaries) == 4 + supplementaries = { + ds.facets['short_name']: ds + for ds in dataset.supplementaries + } + assert supplementaries['areacella'].facets['mip'] == 'fx' + assert supplementaries['areacello'].facets['mip'] == 'Ofx' + assert supplementaries['sftlf'].facets['mip'] == 'fx' + assert supplementaries['sftof'].facets['mip'] == 'Ofx' -def test_fx_list_mip_search_cmip6(tmp_path, patched_datafinder, config_user): +def test_fx_list_mip_search_cmip6(tmp_path, patched_datafinder, session): """Test mip tables search for list of different fx variables.""" content = dedent(""" preprocessors: @@ -3023,7 +2578,7 @@ def test_fx_list_mip_search_cmip6(tmp_path, patched_datafinder, config_user): - {dataset: CanESM5} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -3038,18 +2593,21 @@ def test_fx_list_mip_search_cmip6(tmp_path, patched_datafinder, config_user): assert len(settings) == 1 assert settings['operator'] == 'mean' - # Check add_fx_variables - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 4 - assert '_fx_' in fx_variables['areacella']['filename'].name - assert '_Ofx_' in fx_variables['areacello']['filename'].name - assert '_fx_' in fx_variables['sftlf']['filename'].name - assert '_Ofx_' in fx_variables['sftof']['filename'].name + # Check legacy method of adding supplementary variables + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert len(dataset.supplementaries) == 4 + supplementaries = { + ds.facets['short_name']: ds + for ds in dataset.supplementaries + } + assert supplementaries['areacella'].facets['mip'] == 'fx' + assert supplementaries['areacello'].facets['mip'] == 'Ofx' + assert supplementaries['sftlf'].facets['mip'] == 'fx' + assert supplementaries['sftof'].facets['mip'] == 'Ofx' -def test_fx_vars_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, - config_user): +def test_fx_vars_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, session): TAGS.set_tag_values(TAGS_FOR_TESTING) content = dedent(""" @@ -3077,7 +2635,7 @@ def test_fx_vars_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, - {dataset: CanESM5} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -3091,15 +2649,14 @@ def test_fx_vars_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, settings = product.settings['volume_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 1 - assert '_Omon_' not in fx_variables['volcello']['filename'].name - assert '_Ofx_' in fx_variables['volcello']['filename'].name + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert len(dataset.supplementaries) == 1 + volcello_ds = dataset.supplementaries[0] + assert volcello_ds.facets['mip'] == 'Ofx' -def test_fx_dicts_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, - config_user): +def test_fx_dicts_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, session): content = dedent(""" preprocessors: preproc: @@ -3126,7 +2683,7 @@ def test_fx_dicts_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, - {dataset: CanESM5} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -3140,16 +2697,16 @@ def test_fx_dicts_volcello_in_ofx_cmip6(tmp_path, patched_datafinder, settings = product.settings['volume_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 1 - assert '_Oyr_' in fx_variables['volcello']['filename'][0].name - assert '_piControl_' in fx_variables['volcello']['filename'][0].name - assert '_Omon_' not in fx_variables['volcello']['filename'][0].name + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert len(dataset.supplementaries) == 1 + volcello_ds = dataset.supplementaries[0] + assert volcello_ds.facets['short_name'] == 'volcello' + assert volcello_ds.facets['mip'] == 'Oyr' + assert volcello_ds.facets['exp'] == 'piControl' -def test_fx_vars_list_no_preproc_cmip6(tmp_path, patched_datafinder, - config_user): +def test_fx_vars_list_no_preproc_cmip6(tmp_path, patched_datafinder, session): content = dedent(""" preprocessors: preproc: @@ -3182,7 +2739,7 @@ def test_fx_vars_list_no_preproc_cmip6(tmp_path, patched_datafinder, - {dataset: CanESM5} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -3191,18 +2748,19 @@ def test_fx_vars_list_no_preproc_cmip6(tmp_path, patched_datafinder, assert len(task.ancestors) == 0 assert len(task.products) == 1 product = task.products.pop() + assert len(product.datasets) == 1 + dataset = product.datasets[0] assert product.attributes['short_name'] == 'tos' - assert product.files + assert dataset.files assert 'area_statistics' in product.settings settings = product.settings['area_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert len(fx_variables) == 2 + assert len(dataset.supplementaries) == 2 def test_fx_vars_volcello_in_omon_cmip6(tmp_path, patched_failing_datafinder, - config_user): + session): content = dedent(""" preprocessors: preproc: @@ -3228,7 +2786,7 @@ def test_fx_vars_volcello_in_omon_cmip6(tmp_path, patched_failing_datafinder, - {dataset: CanESM5} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -3242,15 +2800,15 @@ def test_fx_vars_volcello_in_omon_cmip6(tmp_path, patched_failing_datafinder, settings = product.settings['volume_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 1 - assert '_Ofx_' not in fx_variables['volcello']['filename'][0].name - assert '_Omon_' in fx_variables['volcello']['filename'][0].name + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert len(dataset.supplementaries) == 1 + volcello_ds = dataset.supplementaries[0] + assert volcello_ds.facets['mip'] == 'Omon' def test_fx_vars_volcello_in_oyr_cmip6(tmp_path, patched_failing_datafinder, - config_user): + session): content = dedent(""" preprocessors: preproc: @@ -3276,7 +2834,7 @@ def test_fx_vars_volcello_in_oyr_cmip6(tmp_path, patched_failing_datafinder, - {dataset: CanESM5} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -3290,15 +2848,15 @@ def test_fx_vars_volcello_in_oyr_cmip6(tmp_path, patched_failing_datafinder, settings = product.settings['volume_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 1 - assert '_Ofx_' not in fx_variables['volcello']['filename'][0].name - assert '_Oyr_' in fx_variables['volcello']['filename'][0].name + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert len(dataset.supplementaries) == 1 + volcello_ds = dataset.supplementaries[0] + assert volcello_ds.facets['short_name'] == 'volcello' + assert volcello_ds.facets['mip'] == 'Oyr' -def test_fx_vars_volcello_in_fx_cmip5(tmp_path, patched_datafinder, - config_user): +def test_fx_vars_volcello_in_fx_cmip5(tmp_path, patched_datafinder, session): content = dedent(""" preprocessors: preproc: @@ -3322,7 +2880,7 @@ def test_fx_vars_volcello_in_fx_cmip5(tmp_path, patched_datafinder, - {dataset: CanESM2} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -3336,14 +2894,15 @@ def test_fx_vars_volcello_in_fx_cmip5(tmp_path, patched_datafinder, settings = product.settings['volume_statistics'] assert len(settings) == 1 assert settings['operator'] == 'mean' - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 1 - assert '_fx_' in fx_variables['volcello']['filename'].name - assert '_Omon_' not in fx_variables['volcello']['filename'].name + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert len(dataset.supplementaries) == 1 + volcello_ds = dataset.supplementaries[0] + assert volcello_ds.facets['short_name'] == 'volcello' + assert volcello_ds.facets['mip'] == 'fx' -def test_wrong_project(tmp_path, patched_datafinder, config_user): +def test_wrong_project(tmp_path, patched_datafinder, session): content = dedent(""" preprocessors: preproc: @@ -3370,12 +2929,12 @@ def test_wrong_project(tmp_path, patched_datafinder, config_user): msg = ("Unable to load CMOR table (project) 'CMIP7' for variable 'tos' " "with mip 'Omon'") with pytest.raises(RecipeError) as wrong_proj: - get_recipe(tmp_path, content, config_user) + get_recipe(tmp_path, content, session) assert str(wrong_proj.value) == INITIALIZATION_ERROR_MSG assert str(wrong_proj.value.failed_tasks[0].message) == msg -def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, config_user): +def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, session): """Test that error is raised for invalid fx variable.""" TAGS.set_tag_values(TAGS_FOR_TESTING) @@ -3404,22 +2963,22 @@ def test_invalid_fx_var_cmip6(tmp_path, patched_datafinder, config_user): - {dataset: CanESM5} scripts: null """) - msg = ("Requested fx variable 'wrong_fx_variable' not available in any " - "CMOR table") + msg = ("Preprocessor function 'area_statistics' does not support " + "supplementary variable 'wrong_fx_variable'") with pytest.raises(RecipeError) as rec_err_exp: - get_recipe(tmp_path, content, config_user) + get_recipe(tmp_path, content, session) assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG assert msg in rec_err_exp.value.failed_tasks[0].message -def test_ambiguous_fx_var_cmip6(tmp_path, patched_datafinder, config_user): +def test_ambiguous_fx_var_cmip6(tmp_path, patched_datafinder, session): """Test that error is raised for fx files available in multiple mips.""" TAGS.set_tag_values(TAGS_FOR_TESTING) content = dedent(""" preprocessors: preproc: - area_statistics: + volume_statistics: operator: mean fx_variables: volcello: @@ -3441,17 +3000,17 @@ def test_ambiguous_fx_var_cmip6(tmp_path, patched_datafinder, config_user): scripts: null """) msg = ("Requested fx variable 'volcello' for dataset 'CanESM5' of project " - "'CMIP6' is available in more than one CMOR table for 'CMIP6': " + "'CMIP6' is available in more than one CMOR MIP table for 'CMIP6': " "['Odec', 'Ofx', 'Omon', 'Oyr']") with pytest.raises(RecipeError) as rec_err_exp: - get_recipe(tmp_path, content, config_user) + get_recipe(tmp_path, content, session) assert str(rec_err_exp.value) == INITIALIZATION_ERROR_MSG assert msg in rec_err_exp.value.failed_tasks[0].message def test_unique_fx_var_in_multiple_mips_cmip6(tmp_path, patched_failing_datafinder, - config_user): + session): """Test that no error is raised for fx files available in one mip.""" TAGS.set_tag_values(TAGS_FOR_TESTING) @@ -3479,7 +3038,7 @@ def test_unique_fx_var_in_multiple_mips_cmip6(tmp_path, - {dataset: CanESM5} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -3494,19 +3053,19 @@ def test_unique_fx_var_in_multiple_mips_cmip6(tmp_path, assert len(settings) == 1 assert settings['mask_out'] == 'ice' - # Check add_fx_variables + # Check legacy method of adding supplementary variables # Due to failing datafinder, only files in LImon are found even though # sftgif is available in the tables fx, IyrAnt, IyrGre and LImon - fx_variables = product.settings['add_fx_variables']['fx_variables'] - assert isinstance(fx_variables, dict) - assert len(fx_variables) == 1 - sftgif_files = fx_variables['sftgif']['filename'] - assert isinstance(sftgif_files, list) - assert len(sftgif_files) == 1 - assert '_LImon_' in sftgif_files[0].name + assert len(product.datasets) == 1 + dataset = product.datasets[0] + assert len(dataset.supplementaries) == 1 + sftgif_ds = dataset.supplementaries[0] + assert sftgif_ds.facets['short_name'] == 'sftgif' + assert sftgif_ds.facets['mip'] == 'LImon' + assert len(sftgif_ds.files) == 1 -def test_multimodel_mask(tmp_path, patched_datafinder, config_user): +def test_multimodel_mask(tmp_path, patched_datafinder, session): """Test ``mask_multimodel``.""" content = dedent(""" preprocessors: @@ -3530,7 +3089,7 @@ def test_multimodel_mask(tmp_path, patched_datafinder, config_user): - {dataset: HadGEM2-ES} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) # Check generated tasks assert len(recipe.tasks) == 1 @@ -3544,7 +3103,7 @@ def test_multimodel_mask(tmp_path, patched_datafinder, config_user): assert product.settings['mask_multimodel'] == {} -def test_obs4mips_case_correct(tmp_path, patched_datafinder, config_user): +def test_obs4mips_case_correct(tmp_path, patched_datafinder, session): """Test that obs4mips is corrected to obs4MIPs.""" content = dedent(""" diagnostics: @@ -3562,43 +3121,12 @@ def test_obs4mips_case_correct(tmp_path, patched_datafinder, config_user): version: 1, tier: 1, level: 1} scripts: null """) - recipe = get_recipe(tmp_path, content, config_user) - variable = recipe.diagnostics['diagnostic_name']['preprocessor_output'][ - 'tas'][0] - assert variable['project'] == 'obs4MIPs' - - -def test_write_filled_recipe(tmp_path, patched_datafinder, config_user): - - content = dedent(""" - diagnostics: - diagnostic_name: - variables: - tas: - project: CMIP5 - mip: Amon - exp: historical - ensemble: r1i1p1 - timerange: '*' - additional_datasets: - - {dataset: BNU-ESM} - scripts: null - """) + recipe = get_recipe(tmp_path, content, session) + dataset = recipe.datasets[0] + assert dataset['project'] == 'obs4MIPs' - recipe = get_recipe(tmp_path, content, config_user) - run_dir = config_user['run_dir'] - if not os.path.exists(run_dir): - os.makedirs(run_dir) - - recipe._updated_recipe = deepcopy(recipe._raw_recipe) - nested_update(recipe._updated_recipe, 'timerange', - '1990/2019', in_place=True) - esmvalcore._recipe.recipe.Recipe.write_filled_recipe(recipe) - assert os.path.isfile(os.path.join(run_dir, 'recipe_test_filled.yml')) - - -def test_recipe_run(tmp_path, patched_datafinder, config_user, mocker): +def test_recipe_run(tmp_path, patched_datafinder, session, mocker): content = dedent(""" diagnostics: diagnostic_name: @@ -3612,14 +3140,14 @@ def test_recipe_run(tmp_path, patched_datafinder, config_user, mocker): - {dataset: BNU-ESM} scripts: null """) - config_user['download_dir'] = tmp_path / 'download_dir' - config_user['offline'] = False + session['download_dir'] = tmp_path / 'download_dir' + session['offline'] = False mocker.patch.object(esmvalcore._recipe.recipe.esgf, 'download', create_autospec=True) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) recipe.tasks.run = mocker.Mock() recipe.write_filled_recipe = mocker.Mock() @@ -3627,62 +3155,53 @@ def test_recipe_run(tmp_path, patched_datafinder, config_user, mocker): recipe.run() esmvalcore._recipe.recipe.esgf.download.assert_called_once_with( - set(), config_user['download_dir']) + set(), session['download_dir']) recipe.tasks.run.assert_called_once_with( - max_parallel_tasks=config_user['max_parallel_tasks']) + max_parallel_tasks=session['max_parallel_tasks']) recipe.write_filled_recipe.assert_called_once() recipe.write_html_summary.assert_called_once() -@patch('esmvalcore._recipe.check.data_availability', autospec=True) -def test_dataset_to_file_regular_var(mock_data_availability, - patched_datafinder, config_user): - """Test ``_dataset_to_file`` with regular variable.""" +def test_representative_dataset_regular_var(patched_datafinder, session): + """Test ``_representative_dataset`` with regular variable.""" variable = { 'dataset': 'ICON', - 'end_year': 2000, 'exp': 'atm_amip-rad_R2B4_r1i1p1f1', 'frequency': 'mon', 'mip': 'Amon', 'original_short_name': 'tas', 'project': 'ICON', 'short_name': 'tas', - 'start_year': 1990, 'timerange': '1990/2000', 'var_type': 'atm_2d_ml', } - filename = _dataset_to_file(variable, config_user) + dataset = Dataset(**variable) + dataset.session = session + filename = _representative_dataset(dataset).files[0] path = Path(filename) assert path.name == 'atm_amip-rad_R2B4_r1i1p1f1_atm_2d_ml_1990_1999.nc' - mock_data_availability.assert_called_once() -@patch('esmvalcore._recipe.check.data_availability', autospec=True) -@patch('esmvalcore._recipe.recipe._get_input_files', autospec=True) -def test_dataset_to_file_derived_var(mock_get_input_files, - mock_data_availability, config_user): - """Test ``_dataset_to_file`` with derived variable.""" - mock_get_input_files.side_effect = [ - ([], []), - ([sentinel.out_file], [sentinel.globs]), - ] +@pytest.mark.parametrize('force_derivation', [True, False]) +def test_representative_dataset_derived_var(patched_datafinder, session, + force_derivation): + """Test ``_representative_dataset`` with derived variable.""" variable = { 'dataset': 'ICON', 'derive': True, - 'end_year': 2000, 'exp': 'atm_amip-rad_R2B4_r1i1p1f1', - 'force_derivation': True, + 'force_derivation': force_derivation, 'frequency': 'mon', 'mip': 'Amon', 'original_short_name': 'alb', 'project': 'ICON', 'short_name': 'alb', - 'start_year': 1990, 'timerange': '1990/2000', + 'var_type': 'atm_2d_ml', } - filename = _dataset_to_file(variable, config_user) - assert filename == sentinel.out_file - assert mock_get_input_files.call_count == 2 + dataset = Dataset(**variable) + dataset.session = session + representative_dataset = _representative_dataset(dataset) expect_required_var = { # Added by get_required @@ -3690,13 +3209,11 @@ def test_dataset_to_file_derived_var(mock_get_input_files, # Already present in variable 'dataset': 'ICON', 'derive': True, - 'end_year': 2000, 'exp': 'atm_amip-rad_R2B4_r1i1p1f1', - 'force_derivation': True, + 'force_derivation': force_derivation, 'frequency': 'mon', 'mip': 'Amon', 'project': 'ICON', - 'start_year': 1990, 'timerange': '1990/2000', # Added by _add_cmor_info 'long_name': 'Surface Downwelling Clear-Sky Shortwave Radiation', @@ -3708,16 +3225,20 @@ def test_dataset_to_file_derived_var(mock_get_input_files, # Added by _add_extra_facets 'var_type': 'atm_2d_ml', } - mock_get_input_files.assert_called_with(expect_required_var, config_user) - mock_data_availability.assert_called_once() + if force_derivation: + expected_dataset = Dataset(**expect_required_var) + expected_dataset.session = session + else: + expected_dataset = dataset + assert representative_dataset == expected_dataset -def test_get_derive_input_variables(patched_datafinder, config_user): + +def test_get_derive_input_variables(patched_datafinder, session): """Test ``_get_derive_input_variables``.""" - variables = [{ + alb_facets = { 'dataset': 'ICON', 'derive': True, - 'end_year': 2000, 'exp': 'atm_amip-rad_R2B4_r1i1p1f1', 'force_derivation': True, 'frequency': 'mon', @@ -3725,66 +3246,63 @@ def test_get_derive_input_variables(patched_datafinder, config_user): 'original_short_name': 'alb', 'project': 'ICON', 'short_name': 'alb', - 'start_year': 1990, 'timerange': '1990/2000', - 'variable_group': 'alb_group', - }] - derive_input = _get_derive_input_variables(variables, config_user) - - expected_derive_input = { - 'alb_group_derive_input_rsdscs': [{ - # Added by get_required - 'short_name': 'rsdscs', - # Already present in variables - 'dataset': 'ICON', - 'derive': True, - 'end_year': 2000, - 'exp': 'atm_amip-rad_R2B4_r1i1p1f1', - 'force_derivation': True, - 'frequency': 'mon', - 'mip': 'Amon', - 'project': 'ICON', - 'start_year': 1990, - 'timerange': '1990/2000', - # Added by _add_cmor_info - 'standard_name': - 'surface_downwelling_shortwave_flux_in_air_assuming_clear_sky', - 'long_name': 'Surface Downwelling Clear-Sky Shortwave Radiation', - 'modeling_realm': ['atmos'], - 'original_short_name': 'rsdscs', - 'units': 'W m-2', - # Added by _add_extra_facets - 'var_type': 'atm_2d_ml', - # Added by append - 'variable_group': 'alb_group_derive_input_rsdscs', - }], 'alb_group_derive_input_rsuscs': [{ - # Added by get_required - 'short_name': 'rsuscs', - # Already present in variables - 'dataset': 'ICON', - 'derive': True, - 'end_year': 2000, - 'exp': 'atm_amip-rad_R2B4_r1i1p1f1', - 'force_derivation': True, - 'frequency': 'mon', - 'mip': 'Amon', - 'project': 'ICON', - 'start_year': 1990, - 'timerange': '1990/2000', - # Added by _add_cmor_info - 'standard_name': - 'surface_upwelling_shortwave_flux_in_air_assuming_clear_sky', - 'long_name': 'Surface Upwelling Clear-Sky Shortwave Radiation', - 'modeling_realm': ['atmos'], - 'original_short_name': 'rsuscs', - 'units': 'W m-2', - # Added by _add_extra_facets - 'var_type': 'atm_2d_ml', - # Added by append - 'variable_group': 'alb_group_derive_input_rsuscs', - }], } - assert derive_input == expected_derive_input + alb = Dataset(**alb_facets) + alb.session = session + + rsdscs_facets = { + # Added by get_required + 'short_name': 'rsdscs', + # Already present in variables + 'dataset': 'ICON', + 'derive': True, + 'exp': 'atm_amip-rad_R2B4_r1i1p1f1', + 'force_derivation': True, + 'frequency': 'mon', + 'mip': 'Amon', + 'project': 'ICON', + 'timerange': '1990/2000', + # Added by _add_cmor_info + 'standard_name': + 'surface_downwelling_shortwave_flux_in_air_assuming_clear_sky', + 'long_name': 'Surface Downwelling Clear-Sky Shortwave Radiation', + 'modeling_realm': ['atmos'], + 'original_short_name': 'rsdscs', + 'units': 'W m-2', + # Added by _add_extra_facets + 'var_type': 'atm_2d_ml', + } + rsdscs = Dataset(**rsdscs_facets) + rsdscs.session = session + + rsuscs_facets = { + # Added by get_required + 'short_name': 'rsuscs', + # Already present in variables + 'dataset': 'ICON', + 'derive': True, + 'exp': 'atm_amip-rad_R2B4_r1i1p1f1', + 'force_derivation': True, + 'frequency': 'mon', + 'mip': 'Amon', + 'project': 'ICON', + 'timerange': '1990/2000', + # Added by _add_cmor_info + 'standard_name': + 'surface_upwelling_shortwave_flux_in_air_assuming_clear_sky', + 'long_name': 'Surface Upwelling Clear-Sky Shortwave Radiation', + 'modeling_realm': ['atmos'], + 'original_short_name': 'rsuscs', + 'units': 'W m-2', + # Added by _add_extra_facets + 'var_type': 'atm_2d_ml', + } + rsuscs = Dataset(**rsuscs_facets) + rsuscs.session = session + + alb_derive_input = _get_input_datasets(alb) + assert alb_derive_input == [rsdscs, rsuscs] TEST_DIAG_SELECTION = [ @@ -3799,21 +3317,21 @@ def test_get_derive_input_variables(patched_datafinder, config_user): ({'d1/tas'}, {'d1/tas'}), ({'d1/tas', 'd2/*'}, {'d1/tas', 'd1/s1', 'd2/s1'}), ({'d1/tas', 'd3/s1'}, {'d1/tas', 'd3/s1', 'd1/s1'}), - ({'d4/*', 'd3/s1'}, {'d1/tas', 'd1/s1', 'd2/s1', 'd3/s1', 'd3/s2', - 'd4/s1'}), + ({'d4/*', + 'd3/s1'}, {'d1/tas', 'd1/s1', 'd2/s1', 'd3/s1', 'd3/s2', 'd4/s1'}), ] @pytest.mark.parametrize('diags_to_run,tasks_run', TEST_DIAG_SELECTION) -def test_diag_selection(tmp_path, patched_datafinder, config_user, - diags_to_run, tasks_run): +def test_diag_selection(tmp_path, patched_datafinder, session, diags_to_run, + tasks_run): """Test selection of individual diagnostics via --diagnostics option.""" TAGS.set_tag_values(TAGS_FOR_TESTING) script = tmp_path / 'diagnostic.py' script.write_text('') if diags_to_run is not None: - config_user['diagnostics'] = diags_to_run + session['diagnostics'] = diags_to_run content = dedent(""" diagnostics: @@ -3856,7 +3374,7 @@ def test_diag_selection(tmp_path, patched_datafinder, config_user, ancestors: [d3/s2] """).format(script=script) - recipe = get_recipe(tmp_path, content, config_user) + recipe = get_recipe(tmp_path, content, session) task_names = {task.name for task in recipe.tasks.flatten()} assert tasks_run == task_names diff --git a/tests/unit/cmor/test_fix.py b/tests/unit/cmor/test_fix.py index 5f4e9e78e5..e8f98a3972 100644 --- a/tests/unit/cmor/test_fix.py +++ b/tests/unit/cmor/test_fix.py @@ -1,5 +1,6 @@ """Unit tests for :mod:`esmvalcore.cmor.fix`.""" +from pathlib import Path from unittest import TestCase from unittest.mock import Mock, patch @@ -38,7 +39,7 @@ def test_fix(self): project='project', dataset='model', mip='mip', - output_dir='output_dir', + output_dir=Path('output_dir'), ) self.assertNotEqual(file_returned, self.filename) self.assertEqual(file_returned, 'new_filename') @@ -56,7 +57,7 @@ def test_nofix(self): project='project', dataset='model', mip='mip', - output_dir='output_dir', + output_dir=Path('output_dir'), ) self.assertEqual(file_returned, self.filename) mock_get_fixes.assert_called_once_with( diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 432272dee1..b4077de7e1 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -12,6 +12,7 @@ get_extra_facets, importlib_files, ) +from esmvalcore.dataset import Dataset from esmvalcore.exceptions import RecipeError TEST_DEEP_UPDATE = [ @@ -68,53 +69,53 @@ def test_load_extra_facets(project, extra_facets_dir, expected): def test_get_extra_facets(tmp_path): - - variable = { - 'project': 'test_project', - 'mip': 'test_mip', - 'dataset': 'test_dataset', - 'short_name': 'test_short_name', - } - extra_facets_file = tmp_path / f"{variable['project']}-test.yml" + dataset = Dataset( + **{ + 'project': 'test_project', + 'mip': 'test_mip', + 'dataset': 'test_dataset', + 'short_name': 'test_short_name', + }) + extra_facets_file = tmp_path / f"{dataset['project']}-test.yml" extra_facets_file.write_text( textwrap.dedent(""" {dataset}: {mip}: {short_name}: key: value - """).strip().format(**variable)) + """).strip().format(**dataset.facets)) - extra_facets = get_extra_facets(**variable, extra_facets_dir=(tmp_path, )) + extra_facets = get_extra_facets(dataset, extra_facets_dir=(tmp_path, )) assert extra_facets == {'key': 'value'} def test_get_extra_facets_cmip3(): - - variable = { + dataset = Dataset(**{ 'project': 'CMIP3', 'mip': 'A1', 'short_name': 'tas', 'dataset': 'CM3', - } - extra_facets = get_extra_facets(**variable, extra_facets_dir=tuple()) + }) + extra_facets = get_extra_facets(dataset, extra_facets_dir=tuple()) assert extra_facets == {'institute': ['CNRM', 'INM']} def test_get_extra_facets_cmip5(): - - variable = { - 'project': 'CMIP5', - 'mip': 'Amon', - 'short_name': 'tas', - 'dataset': 'ACCESS1-0', - } - extra_facets = get_extra_facets(**variable, extra_facets_dir=tuple()) + dataset = Dataset( + **{ + 'project': 'CMIP5', + 'mip': 'Amon', + 'short_name': 'tas', + 'dataset': 'ACCESS1-0', + }) + extra_facets = get_extra_facets(dataset, extra_facets_dir=tuple()) assert extra_facets == { - 'institute': ['CSIRO-BOM'], 'product': ['output1', 'output2'] - } + 'institute': ['CSIRO-BOM'], + 'product': ['output1', 'output2'] + } def test_get_project_config(mocker): @@ -184,6 +185,7 @@ def test_load_default_config(monkeypatch, default_config): 'run_diagnostic': True, 'skip_nonexistent': False, 'save_intermediary_cubes': False, + 'use_legacy_supplementaries': None, } directory_attrs = { diff --git a/tests/unit/config/test_config_validator.py b/tests/unit/config/test_config_validator.py index 47da116416..6cd92a1195 100644 --- a/tests/unit/config/test_config_validator.py +++ b/tests/unit/config/test_config_validator.py @@ -8,6 +8,7 @@ _listify_validator, deprecate, validate_bool, + validate_bool_or_none, validate_check_level, validate_diagnostics, validate_float, @@ -20,6 +21,7 @@ validate_string, validate_string_or_none, ) +from esmvalcore.exceptions import InvalidConfigParameter def generate_validator_testcases(valid): @@ -90,6 +92,11 @@ def generate_validator_testcases(valid): for _ in ('1, 2', [1.5, 2.5], [1, 2], (1, 2), np.array((1, 2)))), 'fail': ((_, ValueError) for _ in ('fail', ('a', 1), (1, 2, 3))) }, + { + 'validator': validate_bool_or_none, + 'success': ((None, None), (True, True), (False, False)), + 'fail': (('A', ValueError), (1, ValueError)), + }, { 'validator': validate_int_or_none, 'success': ((None, None), ), @@ -187,11 +194,21 @@ def test_validator_invalid(validator, arg, exception_type): @pytest.mark.parametrize('version', (current_version, '0.0.1', '9.9.9')) def test_deprecate(version): - def test_func(): - pass + def test_func(value): + return value - # This always warns - with pytest.warns(UserWarning): - f = deprecate(test_func, 'test_var', version) + validate = deprecate( + validator=test_func, + option='test_var', + default=None, + version=version, + ) + assert callable(validate) - assert callable(f) + if version != '9.9.9': + with pytest.raises(InvalidConfigParameter): + validate('value') + else: + with pytest.warns(UserWarning): + result = validate('value') + assert result == 'value' diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 0000000000..9b312d26d3 --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,14 @@ +import copy + +import pytest + +from esmvalcore.config import CFG +from esmvalcore.config._config_object import CFG_DEFAULT + + +@pytest.fixture +def session(tmp_path, monkeypatch): + for key, value in CFG_DEFAULT.items(): + monkeypatch.setitem(CFG, key, copy.deepcopy(value)) + monkeypatch.setitem(CFG, 'output_dir', tmp_path / 'esmvaltool_output') + return CFG.start_session('recipe_test') diff --git a/tests/unit/esgf/test_download.py b/tests/unit/esgf/test_download.py index ae571769b0..fd2d50d0f2 100644 --- a/tests/unit/esgf/test_download.py +++ b/tests/unit/esgf/test_download.py @@ -568,6 +568,7 @@ def test_download(mocker, tmp_path, caplog): ] for i, file in enumerate(test_files): file.__str__.return_value = f'file{i}.nc' + file.local_file.return_value.exists.return_value = False file.size = 200 * 10**6 file.__lt__.return_value = False @@ -590,6 +591,7 @@ def test_download_fail(mocker, tmp_path, caplog): ] for i, file in enumerate(test_files): file.__str__.return_value = f'file{i}.nc' + file.local_file.return_value.exists.return_value = False file.size = 100 * 10**6 file.__lt__.return_value = False @@ -613,7 +615,7 @@ def test_download_fail(mocker, tmp_path, caplog): def test_download_noop(caplog): """Test downloading no files.""" - caplog.set_level('INFO') + caplog.set_level('DEBUG') esmvalcore.esgf.download([], dest_folder='/does/not/exist') msg = ("All required data is available locally," diff --git a/tests/unit/local/test_time.py b/tests/unit/local/test_time.py index 257cf0ed4e..70e01d70a2 100644 --- a/tests/unit/local/test_time.py +++ b/tests/unit/local/test_time.py @@ -6,7 +6,7 @@ _dates_to_timerange, _get_start_end_date, _get_start_end_year, - _get_timerange_from_years, + _replace_years_with_timerange, _truncate_dates, ) @@ -146,7 +146,7 @@ def test_get_timerange_from_years(): 'start_year': 2000, 'end_year': 2002} - _get_timerange_from_years(variable) + _replace_years_with_timerange(variable) assert 'start_year' not in variable assert 'end_year' not in variable @@ -160,7 +160,7 @@ def test_get_timerange_from_start_year(): 'start_year': 2000 } - _get_timerange_from_years(variable) + _replace_years_with_timerange(variable) assert 'start_year' not in variable assert variable['timerange'] == '2000/2000' @@ -173,7 +173,7 @@ def test_get_timerange_from_end_year(): 'end_year': 2002 } - _get_timerange_from_years(variable) + _replace_years_with_timerange(variable) assert 'end_year' not in variable assert variable['timerange'] == '2002/2002' diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 476f887e2b..52f72554e4 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -16,7 +16,7 @@ import esmvalcore.preprocessor._multimodel as mm from esmvalcore.iris_helpers import date2num from esmvalcore.preprocessor import multi_model_statistics -from esmvalcore.preprocessor._ancillary_vars import add_ancillary_variable +from esmvalcore.preprocessor._supplementary_vars import add_ancillary_variable SPAN_OPTIONS = ('overlap', 'full') diff --git a/tests/unit/preprocessor/_other/test_other.py b/tests/unit/preprocessor/_other/test_other.py index 5795c56db2..ab98895c02 100644 --- a/tests/unit/preprocessor/_other/test_other.py +++ b/tests/unit/preprocessor/_other/test_other.py @@ -47,17 +47,19 @@ def test_clip(self): def test_group_products_string_list(): products = [ PreprocessorFile( + filename='A_B.nc', attributes={ 'project': 'A', 'dataset': 'B', - 'filename': 'A_B.nc'}, - settings={}), + }, + ), PreprocessorFile( + filename='A_C.nc', attributes={ 'project': 'A', 'dataset': 'C', - 'filename': 'A_C.nc'}, - settings={}) + } + ), ] grouped_by_string = _group_products(products, 'project') grouped_by_list = _group_products(products, ['project']) diff --git a/tests/unit/preprocessor/_weighting/test_weighting_landsea_fraction.py b/tests/unit/preprocessor/_weighting/test_weighting_landsea_fraction.py index 71177ec5e3..3e90fee118 100644 --- a/tests/unit/preprocessor/_weighting/test_weighting_landsea_fraction.py +++ b/tests/unit/preprocessor/_weighting/test_weighting_landsea_fraction.py @@ -58,10 +58,10 @@ LAND_FRACTION = [ (CUBE_3, None, [ 'Ancillary variables land/sea area fraction not found in cube. ' - 'Check fx_file availability.']), + 'Check ancillary data availability.']), (CUBE_4, None, [ 'Ancillary variables land/sea area fraction not found in cube. ' - 'Check fx_file availability.']), + 'Check ancillary data availability.']), (CUBE_ANCILLARY_3, FRAC_SFTLF, []), (CUBE_ANCILLARY_4, FRAC_SFTOF, []) ] diff --git a/tests/unit/preprocessor/test_error_logging.py b/tests/unit/preprocessor/test_error_logging.py index 027d14ef6c..adbedc7cd2 100644 --- a/tests/unit/preprocessor/test_error_logging.py +++ b/tests/unit/preprocessor/test_error_logging.py @@ -1,5 +1,6 @@ """Unit tests for error logging of :mod:`esmvalcore.preprocessor`.""" +from pathlib import Path from unittest import mock import pytest @@ -45,7 +46,7 @@ def assert_error_call_ok(mock_logger): KWARGS = {'test': 42, 'list': ['a', 'b']} -PREPROC_FILE = PreprocessorFile({'filename': 'a'}, {}) +PREPROC_FILE = PreprocessorFile(Path('a')) TEST_ITEMS_SHORT = [ # Scalars PREPROC_FILE, @@ -246,22 +247,3 @@ class MockAncestor(): def __init__(self, filename): """Initialize mock ancestor.""" self.filename = filename - - -def test_input_files_for_log(): - """Test :meth:`PreprocessorFile._input_files_for_log`.""" - ancestors = [ - MockAncestor('a.nc'), - MockAncestor('b.nc'), - ] - preproc_file = PreprocessorFile({'filename': 'p.nc'}, {}, - ancestors=ancestors) - - assert preproc_file._input_files == ['a.nc', 'b.nc'] - assert preproc_file.files == ['a.nc', 'b.nc'] - assert preproc_file._input_files_for_log() is None - - preproc_file.files = ['c.nc', 'd.nc'] - assert preproc_file._input_files == ['a.nc', 'b.nc'] - assert preproc_file.files == ['c.nc', 'd.nc'] - assert preproc_file._input_files_for_log() == ['a.nc', 'b.nc'] diff --git a/tests/unit/preprocessor/test_preprocessor_file.py b/tests/unit/preprocessor/test_preprocessor_file.py index 0e29369bcd..df731e5655 100644 --- a/tests/unit/preprocessor/test_preprocessor_file.py +++ b/tests/unit/preprocessor/test_preprocessor_file.py @@ -1,5 +1,6 @@ """Unit tests for :class:`esmvalcore.preprocessor.PreprocessorFile`.""" +from pathlib import Path from unittest import mock import pytest @@ -8,7 +9,7 @@ from esmvalcore.preprocessor import PreprocessorFile ATTRIBUTES = { - 'filename': 'file.nc', + 'filename': Path('file.nc'), 'standard_name': 'precipitation', 'long_name': 'Precipitation', 'short_name': 'pr', @@ -28,7 +29,11 @@ def product(): units='K', attributes={'frequency': 'day'}, ) - product = PreprocessorFile(attributes=ATTRIBUTES, settings={}) + product = PreprocessorFile( + filename=Path('file.nc'), + attributes={k: v for k, v in ATTRIBUTES.items() if k != 'filename'}, + settings={}, + ) product._cubes = CubeList([cube, cube, cube]) return product @@ -47,7 +52,7 @@ def test_update_attributes(product): product._update_attributes() assert product.attributes == { - 'filename': 'file.nc', + 'filename': Path('file.nc'), 'standard_name': 'air_temperature', 'long_name': 'Near-Surface Air Temperature', 'short_name': 'tas', @@ -72,7 +77,7 @@ def test_update_attributes_empty_names(product, name, cube_property, product._update_attributes() expected_attributes = { - 'filename': 'file.nc', + 'filename': Path('file.nc'), 'standard_name': 'air_temperature', 'long_name': 'Near-Surface Air Temperature', 'short_name': 'tas', @@ -90,7 +95,7 @@ def test_update_attributes_empty_frequency(product): product._update_attributes() assert product.attributes == { - 'filename': 'file.nc', + 'filename': Path('file.nc'), 'standard_name': 'air_temperature', 'long_name': 'Near-Surface Air Temperature', 'short_name': 'tas', @@ -107,7 +112,7 @@ def test_update_attributes_no_frequency(product): product._update_attributes() assert product.attributes == { - 'filename': 'file.nc', + 'filename': Path('file.nc'), 'standard_name': 'air_temperature', 'long_name': 'Near-Surface Air Temperature', 'short_name': 'tas', diff --git a/tests/unit/preprocessor/test_runner.py b/tests/unit/preprocessor/test_runner.py index 4bc65c5b98..b1f6866a5f 100644 --- a/tests/unit/preprocessor/test_runner.py +++ b/tests/unit/preprocessor/test_runner.py @@ -1,17 +1,59 @@ -from esmvalcore.preprocessor import (DEFAULT_ORDER, MULTI_MODEL_FUNCTIONS, - _get_itype) +from pathlib import Path + +import iris.cube +import pytest + +import esmvalcore.preprocessor def test_first_argument_name(): """Check that the input type of all preprocessor functions is valid.""" valid_itypes = ('file', 'files', 'cube', 'cubes', 'products', 'input_products') - for step in DEFAULT_ORDER: - itype = _get_itype(step) + for step in esmvalcore.preprocessor.DEFAULT_ORDER: + itype = esmvalcore.preprocessor._get_itype(step) assert itype in valid_itypes, ( "Invalid preprocessor function definition {}, first argument " "should be one of {} but is {}".format(step, valid_itypes, itype)) def test_multi_model_exist(): - assert MULTI_MODEL_FUNCTIONS.issubset(set(DEFAULT_ORDER)) + assert esmvalcore.preprocessor.MULTI_MODEL_FUNCTIONS.issubset( + set(esmvalcore.preprocessor.DEFAULT_ORDER)) + + +@pytest.mark.parametrize('debug', [False, True]) +def test_preprocess_debug(mocker, debug): + in_cube = iris.cube.Cube([1], var_name='tas') + out_cube = iris.cube.Cube([2], var_name='tas') + + items = [in_cube] + result = [out_cube] + step = 'annual_statistics' + input_files = [Path('/path/to/input.nc')] + output_file = Path('/path/to/output.nc') + + mocker.patch.object( + esmvalcore.preprocessor, + 'annual_statistics', + create_autospec=True, + __name__='annual_statistics', + return_value=out_cube, + ) + mocker.patch.object(esmvalcore.preprocessor, 'save', create_autospec=True) + + esmvalcore.preprocessor.preprocess( + items, + step, + input_files=input_files, + output_file=output_file, + debug=debug, + operator='mean', + ) + esmvalcore.preprocessor.annual_statistics.assert_called_with( + in_cube, operator='mean') + if debug: + esmvalcore.preprocessor.save.assert_called_with( + result, '/path/to/output/00_annual_statistics.nc') + else: + esmvalcore.preprocessor.save.assert_not_called() diff --git a/tests/unit/recipe/test_from_datasets.py b/tests/unit/recipe/test_from_datasets.py index 253bc4e8b2..62f83eae34 100644 --- a/tests/unit/recipe/test_from_datasets.py +++ b/tests/unit/recipe/test_from_datasets.py @@ -7,12 +7,42 @@ _group_ensemble_names, _group_identical_facets, _move_one_level_up, + _to_frozen, datasets_to_recipe, ) from esmvalcore.dataset import Dataset from esmvalcore.exceptions import RecipeError +def test_to_frozen(): + data = { + 'abc': 'x', + 'a': { + 'b': [ + 'd', + 'c', + ], + }, + } + + result = _to_frozen(data) + expected = ( + ( + 'a', + (( + 'b', + ( + 'c', + 'd', + ), + ), ), + ), + ('abc', 'x'), + ) + + assert result == expected + + def test_datasets_to_recipe(): dataset = Dataset( short_name='tas', @@ -27,19 +57,19 @@ def test_datasets_to_recipe(): recipe_txt = textwrap.dedent(""" datasets: - - {dataset: 'dataset1'} + - dataset: 'dataset1' diagnostics: diagnostic1: variables: tas: additional_datasets: - - {dataset: 'dataset2'} + - dataset: 'dataset2' pr: {} diagnostic2: variables: tas: {} additional_datasets: - - {dataset: 'dataset3'} + - dataset: 'dataset3' """) recipe = yaml.safe_load(recipe_txt) @@ -81,14 +111,14 @@ def test_update_datasets_in_recipe(): assert datasets_to_recipe([dataset], recipe=existing_recipe) == recipe -def test_ancillary_datasets_to_recipe(): +def test_supplementary_datasets_to_recipe(): dataset = Dataset( short_name='ta', dataset='dataset1', ) dataset['diagnostic'] = 'diagnostic1' dataset['variable_group'] = 'group1' - dataset.add_ancillary(short_name='areacella') + dataset.add_supplementary(short_name='areacella') recipe_txt = textwrap.dedent(""" datasets: @@ -98,7 +128,7 @@ def test_ancillary_datasets_to_recipe(): variables: group1: short_name: 'ta' - ancillary_variables: + supplementary_variables: - short_name: areacella """) recipe = yaml.safe_load(recipe_txt) diff --git a/tests/unit/recipe/test_io.py b/tests/unit/recipe/test_io.py new file mode 100644 index 0000000000..50fae1a396 --- /dev/null +++ b/tests/unit/recipe/test_io.py @@ -0,0 +1,17 @@ +from esmvalcore._recipe import _io + + +def test_copy_dict(): + a = {'a': 1} + b = {'b': a, 'c': a} + result = _io._copy(b) + assert result['b'] == result['c'] + assert result['b'] is not result['c'] + + +def test_copy_list(): + a = ['a'] + b = {'b': a, 'c': a} + result = _io._copy(b) + assert result['b'] == result['c'] + assert result['b'] is not result['c'] diff --git a/tests/unit/recipe/test_recipe.py b/tests/unit/recipe/test_recipe.py index 2a9411e855..603ab47ae7 100644 --- a/tests/unit/recipe/test_recipe.py +++ b/tests/unit/recipe/test_recipe.py @@ -1,4 +1,3 @@ -from collections import defaultdict from pathlib import Path from unittest import mock @@ -8,7 +7,9 @@ import pytest import esmvalcore._recipe.recipe as _recipe +import esmvalcore.config import esmvalcore.experimental.recipe_output +from esmvalcore.dataset import Dataset from esmvalcore.esgf._download import ESGFFile from esmvalcore.exceptions import RecipeError from tests import PreprocessorFile @@ -19,130 +20,42 @@ class MockRecipe(_recipe.Recipe): def __init__(self, cfg, diagnostics): """Simple constructor used for testing.""" - self._cfg = cfg + self.session = cfg self.diagnostics = diagnostics -class TestRecipe: - - def test_expand_ensemble(self): - - datasets = [ - { - 'dataset': 'XYZ', - 'ensemble': 'r(1:2)i(2:3)p(3:4)', - }, - ] - - expanded = _recipe.Recipe._expand_tag(datasets, 'ensemble') - - ensembles = [ - 'r1i2p3', - 'r1i2p4', - 'r1i3p3', - 'r1i3p4', - 'r2i2p3', - 'r2i2p4', - 'r2i3p3', - 'r2i3p4', - ] - for i, ensemble in enumerate(ensembles): - assert expanded[i] == {'dataset': 'XYZ', 'ensemble': ensemble} - - def test_expand_subexperiment(self): - - datasets = [ - { - 'dataset': 'XYZ', - 'sub_experiment': 's(1998:2005)', - }, - ] - - expanded = _recipe.Recipe._expand_tag(datasets, 'sub_experiment') - - subexperiments = [ - 's1998', - 's1999', - 's2000', - 's2001', - 's2002', - 's2003', - 's2004', - 's2005', - ] - for i, subexperiment in enumerate(subexperiments): - assert expanded[i] == { - 'dataset': 'XYZ', - 'sub_experiment': subexperiment - } - - def test_expand_ensemble_nolist(self): - - datasets = [ - { - 'dataset': 'XYZ', - 'ensemble': ['r1i1p1', 'r(1:2)i1p1'] - }, - ] - - with pytest.raises(RecipeError): - _recipe.Recipe._expand_tag(datasets, 'ensemble') - - VAR_A = {'dataset': 'A'} VAR_A_REF_A = {'dataset': 'A', 'reference_dataset': 'A'} VAR_A_REF_B = {'dataset': 'A', 'reference_dataset': 'B'} TEST_ALLOW_SKIPPING = [ - ([], VAR_A, {}, False), - ([], VAR_A, { + (VAR_A, { 'skip_nonexistent': False }, False), - ([], VAR_A, { + (VAR_A, { 'skip_nonexistent': True }, True), - ([], VAR_A_REF_A, {}, False), - ([], VAR_A_REF_A, { + (VAR_A_REF_A, { 'skip_nonexistent': False }, False), - ([], VAR_A_REF_A, { + (VAR_A_REF_A, { 'skip_nonexistent': True }, False), - ([], VAR_A_REF_B, {}, False), - ([], VAR_A_REF_B, { + (VAR_A_REF_B, { 'skip_nonexistent': False }, False), - ([], VAR_A_REF_B, { + (VAR_A_REF_B, { 'skip_nonexistent': True }, True), - (['A'], VAR_A, {}, False), - (['A'], VAR_A, { - 'skip_nonexistent': False - }, False), - (['A'], VAR_A, { - 'skip_nonexistent': True - }, False), - (['A'], VAR_A_REF_A, {}, False), - (['A'], VAR_A_REF_A, { - 'skip_nonexistent': False - }, False), - (['A'], VAR_A_REF_A, { - 'skip_nonexistent': True - }, False), - (['A'], VAR_A_REF_B, {}, False), - (['A'], VAR_A_REF_B, { - 'skip_nonexistent': False - }, False), - (['A'], VAR_A_REF_B, { - 'skip_nonexistent': True - }, False), ] -@pytest.mark.parametrize('ancestors,var,cfg,out', TEST_ALLOW_SKIPPING) -def test_allow_skipping(ancestors, var, cfg, out): +@pytest.mark.parametrize('var,cfg,out', TEST_ALLOW_SKIPPING) +def test_allow_skipping(var, cfg, out): """Test ``_allow_skipping``.""" - result = _recipe._allow_skipping(ancestors, var, cfg) + dataset = Dataset(**var) + dataset.session = cfg + result = _recipe._allow_skipping(dataset) assert result is out @@ -161,13 +74,18 @@ def test_resume_preprocessor_tasks(mocker, tmp_path): # Create a mock recipe recipe = mocker.create_autospec(_recipe.Recipe, instance=True) - recipe._cfg = { - 'resume_from': [str(prev_output)], - 'preproc_dir': '/path/to/recipe_test_20210101_000000/preproc', - } + + class Session(dict): + pass + + session = Session(resume_from=[prev_output]) + session.preproc_dir = Path('/path/to/recipe_test_20210101_000000/preproc') + recipe.session = session # Create a very simplified list of datasets - diagnostic = {'preprocessor_output': {'tas': [{'short_name': 'tas'}]}} + diagnostic = { + 'datasets': [Dataset(short_name='tas', variable_group='tas')], + } # Create tasks tasks, failed = _recipe.Recipe._create_preprocessor_tasks( @@ -191,7 +109,7 @@ def create_esgf_search_results(): file0 = ESGFFile([ pyesgf.search.results.FileResult( json={ - 'dataset_id': dataset_id, + 'dataset_id': dataset_id, 'dataset_id_template_': [dataset_id_template], 'project': ['CMIP6'], 'size': @@ -237,40 +155,20 @@ def create_esgf_search_results(): @pytest.mark.parametrize("local_availability", ['all', 'partial', 'none']) -@pytest.mark.parametrize('already_downloaded', [True, False]) -def test_search_esgf(mocker, tmp_path, local_availability, already_downloaded): - - rootpath = tmp_path / 'local' - download_dir = tmp_path / 'download_dir' +def test_schedule_for_download(monkeypatch, tmp_path, local_availability): + """Test that `_schedule_for_download` updates DOWNLOAD_FILES.""" esgf_files = create_esgf_search_results() - - # ESGF files may have been downloaded previously, but not have - # been found if the download_dir is not configured as a rootpath - if already_downloaded: - for file in esgf_files: - local_path = file.local_file(download_dir) - local_path.parent.mkdir(parents=True, exist_ok=True) - local_path.touch() + download_dir = tmp_path / 'download_dir' + local_dir = Path('/local_dir') # Local files can cover the entire period, part of it, or nothing local_file_options = { - 'all': [f.local_file(rootpath) for f in esgf_files], - 'partial': [esgf_files[1].local_file(rootpath)], + 'all': [f.local_file(local_dir) for f in esgf_files], + 'partial': [esgf_files[1].local_file(local_dir)], 'none': [], } local_files = local_file_options[local_availability] - mocker.patch.object(_recipe, - 'find_files', - autospec=True, - return_value=(list(local_files), [])) - mocker.patch.object( - _recipe.esgf, - 'find_files', - autospec=True, - return_value=esgf_files, - ) - variable = { 'project': 'CMIP6', 'mip': 'Amon', @@ -283,68 +181,24 @@ def test_search_esgf(mocker, tmp_path, local_availability, already_downloaded): 'timerange': '1850/1851', 'alias': 'CMIP6_EC-Eeath3_tas', } - - config_user = { - 'rootpath': None, - 'drs': None, - 'offline': False, - 'always_search_esgf': False, - 'download_dir': download_dir - } - input_files = _recipe._get_input_files(variable, config_user)[0] - - download_files = [ - f.local_file(download_dir) for f in esgf_files - ] - - expected = { + dataset = Dataset(**variable) + files = { 'all': local_files, - 'partial': local_files + download_files[:1], - 'none': download_files, - } - assert input_files == expected[local_availability] - - -@pytest.mark.parametrize('timerange', ['*', '185001/*', '*/185112']) -def test_search_esgf_timerange(mocker, tmp_path, timerange): - - download_dir = tmp_path / 'download_dir' - esgf_files = create_esgf_search_results() - - mocker.patch.object(_recipe, - 'find_files', - autospec=True, - return_value=[]) - mocker.patch.object( - _recipe.esgf, - 'find_files', - autospec=True, - return_value=esgf_files, - ) - - variable = { - 'project': 'CMIP6', - 'mip': 'Amon', - 'frequency': 'mon', - 'short_name': 'tas', - 'dataset': 'EC.-Earth3', - 'exp': 'historical', - 'ensemble': 'r1i1p1f1', - 'grid': 'gr', - 'timerange': timerange, - 'alias': 'CMIP6_EC-Eeath3_tas', - 'original_short_name': 'tas' + 'partial': local_files + esgf_files[:1], + 'none': esgf_files, } + dataset.session = {'download_dir': download_dir} + dataset.files = list(files[local_availability]) - config_user = { - 'rootpath': None, - 'drs': None, - 'offline': False, - 'download_dir': download_dir + monkeypatch.setattr(_recipe, 'DOWNLOAD_FILES', set()) + _recipe._schedule_for_download([dataset]) + print(esgf_files) + expected = { + 'all': set(), + 'partial': set(esgf_files[:1]), + 'none': set(esgf_files), } - _recipe._update_timerange(variable, config_user) - - assert variable['timerange'] == '185001/185112' + assert _recipe.DOWNLOAD_FILES == expected[local_availability] def test_write_html_summary(mocker, caplog): @@ -365,35 +219,6 @@ def test_write_html_summary(mocker, caplog): mock_recipe.get_output.assert_called_once() -def test_add_fxvar_keys_extra_facets(): - """Test correct addition of extra facets to fx variables.""" - fx_info = {'short_name': 'areacella', 'mip': 'fx'} - variable = {'project': 'ICON', 'dataset': 'ICON'} - extra_facets_dir = tuple() - fx_var = _recipe._add_fxvar_keys(fx_info, variable, extra_facets_dir) - expected_fx_var = { - # Already given by fx_info and variable - 'short_name': 'areacella', - 'mip': 'fx', - 'project': 'ICON', - 'dataset': 'ICON', - # Added by _add_fxvar_keys - 'variable_group': 'areacella', - # Added by _add_cmor_info - 'original_short_name': 'areacella', - 'standard_name': 'cell_area', - 'long_name': 'Grid-Cell Area for Atmospheric Grid Variables', - 'units': 'm2', - 'modeling_realm': ['atmos', 'land'], - 'frequency': 'fx', - # Added by _add_extra_facets - 'latitude': 'grid_latitude', - 'longitude': 'grid_longitude', - 'raw_name': 'cell_area', - } - assert fx_var == expected_fx_var - - def test_multi_model_filename_overlap(): """Test timerange in multi-model filename is correct.""" cube = iris.cube.Cube(np.array([1])) @@ -580,12 +405,6 @@ def test_update_multiproduct_no_product(): assert settings == {} -def test_match_products_no_product(): - variables = [{'var_name': 'var'}] - grouped_products = _recipe._match_products(None, variables) - assert grouped_products == defaultdict(list) - - SCRIPTS_CFG = { 'output_dir': mock.sentinel.output_dir, 'script': mock.sentinel.script, @@ -603,7 +422,7 @@ def test_match_products_no_product(): }}, } TEST_GET_TASKS_TO_RUN = [ - (None, []), + (None, None), ({''}, {''}), ({'wrong_task/*'}, {'wrong_task/*'}), ({'d1/*'}, {'d1/*'}), @@ -626,9 +445,7 @@ def test_match_products_no_product(): TEST_GET_TASKS_TO_RUN) def test_get_tasks_to_run(diags_to_run, tasknames_to_run): """Test ``Recipe._get_tasks_to_run``.""" - cfg = {} - if diags_to_run is not None: - cfg = {'diagnostics': diags_to_run} + cfg = {'diagnostics': diags_to_run} recipe = MockRecipe(cfg, DIAGNOSTICS) tasks_to_run = recipe._get_tasks_to_run() @@ -672,34 +489,6 @@ def test_create_diagnostic_tasks(mock_diag_task, tasks_to_run, tasks_run): assert expected_call in mock_diag_task.mock_calls -def test_differing_timeranges(caplog): - timeranges = set() - timeranges.add('1950/1951') - timeranges.add('1950/1952') - required_variables = [ - { - 'short_name': 'rsdscs', - 'timerange': '1950/1951' - }, - { - 'short_name': 'rsuscs', - 'timerange': '1950/1952' - }, - ] - with pytest.raises(ValueError) as exc: - _recipe._check_differing_timeranges( - timeranges, required_variables) - expected_log = ( - f"Differing timeranges with values {timeranges} " - "found for required variables " - "[{'short_name': 'rsdscs', 'timerange': '1950/1951'}, " - "{'short_name': 'rsuscs', 'timerange': '1950/1952'}]. " - "Set `timerange` to a common value." - ) - - assert expected_log in str(exc.value) - - def test_update_warning_settings_nonaffected_project(): """Test ``_update_warning_settings``.""" settings = {'save': {'filename': 'out.nc'}, 'load': {'filename': 'in.nc'}} @@ -726,3 +515,152 @@ def test_update_warning_settings_step_present(): assert len(settings['load']) == 2 assert settings['load']['filename'] == 'in.nc' assert 'ignore_warnings' in settings['load'] + + +def test_update_regrid_time(): + """Test `_update_regrid_time.""" + dataset = Dataset(frequency='mon') + settings = {'regrid_time': {}} + _recipe._update_regrid_time(dataset, settings) + assert settings == {'regrid_time': {'frequency': 'mon'}} + + +def test_select_dataset_fails(): + dataset = Dataset( + dataset='dataset1', + diagnostic='diagnostic1', + variable_group='tas', + ) + with pytest.raises(RecipeError): + _recipe._select_dataset('dataset2', [dataset]) + + +def test_limit_datasets(): + + datasets = [ + Dataset(dataset='dataset1', alias='dataset1'), + Dataset(dataset='dataset2', alias='dataset2'), + ] + datasets[0].session = {'max_datasets': 1} + + result = _recipe._limit_datasets(datasets, {}) + + assert result == datasets[:1] + + +def test_get_default_settings(mocker): + mocker.patch.object( + _recipe, + '_get_output_file', + autospec=True, + return_value=Path('/path/to/file.nc'), + ) + session = mocker.create_autospec(esmvalcore.config.Session, instance=True) + session.__getitem__.return_value = False + + dataset = Dataset( + short_name='sic', + original_short_name='siconc', + mip='Amon', + project='CMIP6', + ) + dataset.session = session + + settings = _recipe._get_default_settings(dataset) + assert settings == { + 'load': {'callback': 'default'}, + 'remove_supplementary_variables': {}, + 'save': {'compress': False, 'alias': 'sic'}, + 'cleanup': {'remove': ['/path/to/file_fixed']}, + } + + +def test_add_legacy_supplementaries_disabled(): + """Test that `_add_legacy_supplementaries` does nothing when disabled.""" + dataset = Dataset() + dataset.session = {'use_legacy_supplementaries': False} + _recipe._add_legacy_supplementary_datasets(dataset, settings={}) + + +def test_enable_legacy_supplementaries_when_used(mocker, session): + """Test that legacy supplementaries are enabled when used in the recipe.""" + recipe = mocker.create_autospec(_recipe.Recipe, instance=True) + recipe.session = session + recipe._preprocessors = { + 'preproc1': { + 'area_statistics': { + 'operator': 'mean', + 'fx_variables': 'areacella', + } + } + } + session['use_legacy_supplementaries'] = None + _recipe.Recipe._set_use_legacy_supplementaries(recipe) + + assert session['use_legacy_supplementaries'] is True + + +def test_strip_legacy_supplementaries_when_disabled(mocker, session): + """Test that legacy supplementaries are removed when disabled.""" + recipe = mocker.create_autospec(_recipe.Recipe, instance=True) + recipe.session = session + recipe._preprocessors = { + 'preproc1': { + 'area_statistics': { + 'operator': 'mean', + 'fx_variables': 'areacella', + } + } + } + session['use_legacy_supplementaries'] = False + _recipe.Recipe._set_use_legacy_supplementaries(recipe) + + assert session['use_legacy_supplementaries'] is False + assert recipe._preprocessors == { + 'preproc1': { + 'area_statistics': { + 'operator': 'mean', + } + } + } + + +def test_set_version(mocker): + + dataset = Dataset(short_name='tas') + supplementary = Dataset(short_name='areacella') + dataset.supplementaries = [supplementary] + + input_dataset = Dataset(short_name='tas') + file1 = mocker.Mock() + file1.facets = {'version': 'v1'} + file2 = mocker.Mock() + file2.facets = {'version': 'v2'} + input_dataset.files = [file1, file2] + + file3 = mocker.Mock() + file3.facets = {'version': 'v3'} + supplementary.files = [file3] + + _recipe._set_version(dataset, [input_dataset]) + print(dataset) + assert dataset.facets['version'] == ['v1', 'v2'] + assert dataset.supplementaries[0].facets['version'] == 'v3' + + +def test_extract_preprocessor_order(): + profile = { + 'custom_order': True, + 'regrid': { + 'target_grid': '1x1' + }, + 'derive': { + 'long_name': 'albedo at the surface', + 'short_name': 'alb', + 'standard_name': '', + 'units': '1' + }, + } + order = _recipe._extract_preprocessor_order(profile) + assert any(order[i:i + 2] == ('regrid', 'derive') + for i in range(len(order) - 1)) diff --git a/tests/unit/recipe/test_to_datasets.py b/tests/unit/recipe/test_to_datasets.py new file mode 100644 index 0000000000..a2fe0e76d9 --- /dev/null +++ b/tests/unit/recipe/test_to_datasets.py @@ -0,0 +1,386 @@ +import textwrap +from pathlib import Path + +import pytest +import yaml + +from esmvalcore._recipe import to_datasets +from esmvalcore.dataset import Dataset +from esmvalcore.exceptions import RecipeError +from esmvalcore.local import LocalFile + + +def test_from_recipe(session): + recipe_txt = textwrap.dedent(""" + datasets: + - dataset: cccma_cgcm3_1 + ensemble: run1 + exp: historical + frequency: mon + mip: A1 + project: CMIP3 + - dataset: EC-EARTH + ensemble: r1i1p1 + exp: historical + mip: Amon + project: CMIP5 + - dataset: AWI-ESM-1-1-LR + ensemble: r1i1p1f1 + exp: historical + grid: gn + mip: Amon + project: CMIP6 + - dataset: RACMO22E + driver: MOHC-HadGEM2-ES + domain: EUR-11 + ensemble: r1i1p1 + exp: historical + mip: mon + project: CORDEX + - dataset: CERES-EBAF + mip: Amon + project: obs4MIPs + + preprocessors: + preprocessor1: + extract_levels: + levels: 85000 + scheme: nearest + + diagnostics: + diagnostic1: + variables: + ta850: + short_name: ta + preprocessor: preprocessor1 + """) + datasets = Dataset.from_recipe(recipe_txt, session) + + reference = [ + Dataset( + alias='CMIP3', + dataset='cccma_cgcm3_1', + diagnostic='diagnostic1', + ensemble='run1', + exp='historical', + frequency='mon', + mip='A1', + preprocessor='preprocessor1', + project='CMIP3', + recipe_dataset_index=0, + short_name='ta', + variable_group='ta850', + ), + Dataset( + alias='CMIP5', + dataset='EC-EARTH', + diagnostic='diagnostic1', + ensemble='r1i1p1', + exp='historical', + mip='Amon', + preprocessor='preprocessor1', + project='CMIP5', + recipe_dataset_index=1, + short_name='ta', + variable_group='ta850', + ), + Dataset( + alias='CMIP6', + dataset='AWI-ESM-1-1-LR', + diagnostic='diagnostic1', + ensemble='r1i1p1f1', + exp='historical', + grid='gn', + mip='Amon', + preprocessor='preprocessor1', + project='CMIP6', + recipe_dataset_index=2, + short_name='ta', + variable_group='ta850', + ), + Dataset( + alias='CORDEX', + dataset='RACMO22E', + diagnostic='diagnostic1', + driver='MOHC-HadGEM2-ES', + domain='EUR-11', + ensemble='r1i1p1', + exp='historical', + mip='mon', + preprocessor='preprocessor1', + project='CORDEX', + recipe_dataset_index=3, + short_name='ta', + variable_group='ta850', + ), + Dataset( + alias='obs4MIPs', + dataset='CERES-EBAF', + diagnostic='diagnostic1', + mip='Amon', + preprocessor='preprocessor1', + project='obs4MIPs', + recipe_dataset_index=4, + short_name='ta', + variable_group='ta850', + ), + ] + for ref_ds in reference: + ref_ds.session = session + + assert datasets == reference + + +@pytest.mark.parametrize('path_type', [str, Path]) +def test_from_recipe_file(tmp_path, session, path_type): + recipe_file = tmp_path / 'recipe_test.yml' + recipe_txt = textwrap.dedent(""" + datasets: + - dataset: AWI-ESM-1-1-LR + grid: gn + + diagnostics: + diagnostic1: + variables: + tas: + ensemble: r1i1p1f1 + exp: historical + mip: Amon + project: CMIP6 + + """) + recipe_file.write_text(recipe_txt, encoding='utf-8') + datasets = Dataset.from_recipe( + path_type(recipe_file), + session, + ) + assert len(datasets) == 1 + + +def test_from_recipe_dict(session): + recipe_txt = textwrap.dedent(""" + datasets: + - dataset: AWI-ESM-1-1-LR + grid: gn + + diagnostics: + diagnostic1: + variables: + tas: + ensemble: r1i1p1f1 + exp: historical + mip: Amon + project: CMIP6 + + """) + recipe_dict = yaml.safe_load(recipe_txt) + datasets = Dataset.from_recipe(recipe_dict, session) + assert len(datasets) == 1 + + +def test_merge_supplementaries_dataset_takes_priority(session): + recipe_txt = textwrap.dedent(""" + datasets: + - dataset: AWI-ESM-1-1-LR + grid: gn + - dataset: BCC-ESM1 + grid: gn + supplementary_variables: + - short_name: areacella + exp: 1pctCO2 + + preprocessors: + global_mean: + area_statistics: + statistic: mean + + diagnostics: + diagnostic1: + variables: + tas: + ensemble: r1i1p1f1 + exp: historical + mip: Amon + preprocessor: global_mean_land + project: CMIP6 + supplementary_variables: + - short_name: areacella + mip: fx + + """) + + datasets = Dataset.from_recipe(recipe_txt, session) + print(datasets) + assert len(datasets) == 2 + assert all(len(ds.supplementaries) == 1 for ds in datasets) + assert datasets[0].supplementaries[0].facets['exp'] == 'historical' + assert datasets[1].supplementaries[0].facets['exp'] == '1pctCO2' + + +def test_merge_supplementaries_combine_dataset_with_variable(session): + recipe_txt = textwrap.dedent(""" + datasets: + - dataset: AWI-ESM-1-1-LR + grid: gn + supplementary_variables: + - short_name: sftlf + mip: fx + + preprocessors: + global_mean_land: + mask_landsea: + mask_out: sea + area_statistics: + statistic: mean + + diagnostics: + diagnostic1: + variables: + tas: + ensemble: r1i1p1f1 + exp: historical + mip: Amon + preprocessor: global_mean_land + project: CMIP6 + supplementary_variables: + - short_name: areacella + mip: fx + + """) + + datasets = Dataset.from_recipe(recipe_txt, session) + print(datasets) + assert len(datasets) == 1 + assert len(datasets[0].supplementaries) == 2 + assert datasets[0].supplementaries[0].facets['short_name'] == 'areacella' + assert datasets[0].supplementaries[1].facets['short_name'] == 'sftlf' + + +def test_merge_supplementaries_missing_short_name_fails(session): + recipe_txt = textwrap.dedent(""" + diagnostics: + diagnostic1: + variables: + tas: + ensemble: r1i1p1f1 + exp: historical + mip: Amon + project: CMIP6 + supplementary_variables: + - mip: fx + additional_datasets: + - dataset: AWI-ESM-1-1-LR + grid: gn + """) + + with pytest.raises(RecipeError): + Dataset.from_recipe(recipe_txt, session) + + +def test_max_years(session): + recipe_txt = textwrap.dedent(""" + diagnostics: + diagnostic1: + variables: + tas: + ensemble: r1i1p1f1 + exp: historical + mip: Amon + project: CMIP6 + start_year: 2000 + end_year: 2010 + additional_datasets: + - dataset: AWI-ESM-1-1-LR + grid: gn + """) + session['max_years'] = 2 + datasets = Dataset.from_recipe(recipe_txt, session) + assert datasets[0].facets['timerange'] == '2000/2001' + + +@pytest.mark.parametrize('found_files', [True, False]) +def test_dataset_from_files_fails(monkeypatch, found_files): + def from_files(_): + file = LocalFile('/path/to/file') + file.facets = {'facets1': 'value1'} + dataset = Dataset( + dataset='*', + short_name='tas', + ) + dataset.files = [file] if found_files else [] + dataset._file_globs = ['/path/to/tas_*.nc'] + return [dataset] + + monkeypatch.setattr(Dataset, 'from_files', from_files) + + dataset = Dataset( + dataset='*', + short_name='tas', + ) + + with pytest.raises(RecipeError, match="Unable to replace dataset.*"): + to_datasets._dataset_from_files(dataset) + + +def test_fix_cmip5_fx_ensemble(monkeypatch): + def find_files(self): + if self.facets['ensemble'] == 'r0i0p0': + self._files = ['file1.nc'] + + monkeypatch.setattr(Dataset, 'find_files', find_files) + + dataset = Dataset( + dataset='dataset1', + short_name='orog', + mip='fx', + project='CMIP5', + ensemble='r1i1p1', + ) + + to_datasets._fix_cmip5_fx_ensemble(dataset) + + assert dataset['ensemble'] == 'r0i0p0' + + +def test_get_supplementary_short_names(monkeypatch): + def _update_cmor_facets(facets): + facets['modeling_realm'] = 'atmos' + + monkeypatch.setattr( + to_datasets, + '_update_cmor_facets', + _update_cmor_facets, + ) + facets = { + 'short_name': 'tas', + } + result = to_datasets._get_supplementary_short_names(facets, 'mask_landsea') + assert result == ['sftlf'] + + +def test_append_missing_supplementaries(): + supplementaries = [ + { + 'short_name': 'areacella', + }, + ] + facets = { + 'short_name': 'tas', + 'project': 'CMIP6', + 'mip': 'Amon', + } + + settings = { + 'mask_landsea': { + 'mask_out': 'land' + }, + 'area_statistics': { + 'operator': 'mean' + }, + } + + to_datasets._append_missing_supplementaries(supplementaries, facets, + settings) + + short_names = {f['short_name'] for f in supplementaries} + assert short_names == {'areacella', 'sftlf'} diff --git a/tests/unit/task/test_print.py b/tests/unit/task/test_print.py index 0db6b32423..605f017416 100644 --- a/tests/unit/task/test_print.py +++ b/tests/unit/task/test_print.py @@ -5,19 +5,24 @@ import pytest from esmvalcore._task import DiagnosticTask +from esmvalcore.dataset import Dataset from esmvalcore.preprocessor import PreprocessingTask, PreprocessorFile @pytest.fixture def preproc_file(): + dataset = Dataset(short_name='tas') + dataset.files = ['/path/to/input_file.nc'] return PreprocessorFile( - attributes={'filename': '/output/preproc/file.nc'}, + filename='/output/preproc/file.nc', + attributes={'short_name': 'tas'}, settings={ 'extract_levels': { 'scheme': 'linear', 'levels': [95000] }, }, + datasets=[dataset], ) @@ -49,7 +54,8 @@ def test_repr_preproc_task(preproc_task): PreprocessingTask: diag_1/tas order: ['extract_levels', 'save'] PreprocessorFile: /output/preproc/file.nc - {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, + input files: ['/path/to/input_file.nc'] + settings: {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, 'save': {'filename': '/output/preproc/file.nc'}} ancestors: None @@ -93,7 +99,8 @@ def test_repr_simple_tree(preproc_task, diagnostic_task): PreprocessingTask: diag_1/tas order: ['extract_levels', 'save'] PreprocessorFile: /output/preproc/file.nc - {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, + input files: ['/path/to/input_file.nc'] + settings: {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, 'save': {'filename': '/output/preproc/file.nc'}} ancestors: None @@ -136,13 +143,15 @@ def test_repr_full_tree(preproc_task, diagnostic_task): PreprocessingTask: diag_1/tas order: ['extract_levels', 'save'] PreprocessorFile: /output/preproc/file.nc - {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, + input files: ['/path/to/input_file.nc'] + settings: {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, 'save': {'filename': '/output/preproc/file.nc'}} ancestors: PreprocessingTask: diag_1/tas_derive_input_1 order: ['extract_levels', 'save'] PreprocessorFile: /output/preproc/file.nc - {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, + input files: ['/path/to/input_file.nc'] + settings: {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, 'save': {'filename': '/output/preproc/file.nc'}} ancestors: None @@ -150,7 +159,8 @@ def test_repr_full_tree(preproc_task, diagnostic_task): PreprocessingTask: diag_1/tas_derive_input_2 order: ['extract_levels', 'save'] PreprocessorFile: /output/preproc/file.nc - {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, + input files: ['/path/to/input_file.nc'] + settings: {'extract_levels': {'levels': [95000], 'scheme': 'linear'}, 'save': {'filename': '/output/preproc/file.nc'}} ancestors: None diff --git a/tests/unit/test_dataset.py b/tests/unit/test_dataset.py index b8ddab1b0b..5642c88e89 100644 --- a/tests/unit/test_dataset.py +++ b/tests/unit/test_dataset.py @@ -1,4 +1,5 @@ import textwrap +from collections import defaultdict from pathlib import Path import pyesgf @@ -7,21 +8,11 @@ import esmvalcore.dataset import esmvalcore.local from esmvalcore.cmor.check import CheckLevels -from esmvalcore.config import CFG -from esmvalcore.config._config_object import CFG_DEFAULT from esmvalcore.dataset import Dataset from esmvalcore.esgf import ESGFFile from esmvalcore.exceptions import InputFilesNotFound, RecipeError -@pytest.fixture -def session(tmp_path): - CFG.clear() - CFG.update(CFG_DEFAULT) - CFG['output_dir'] = tmp_path / 'esmvaltool_output' - return CFG.start_session('recipe_test') - - def test_repr(): ds = Dataset(short_name='tas', dataset='dataset1') @@ -42,14 +33,14 @@ def test_repr_session(mocker): """).strip() -def test_repr_ancillary(): +def test_repr_supplementary(): ds = Dataset(dataset='dataset1', short_name='tas') - ds.add_ancillary(short_name='areacella') + ds.add_supplementary(short_name='areacella') assert repr(ds) == textwrap.dedent(""" Dataset: {'dataset': 'dataset1', 'short_name': 'tas'} - ancillaries: + supplementaries: {'dataset': 'dataset1', 'short_name': 'areacella'} """).strip() @@ -61,10 +52,10 @@ def test_short_summary(): short_name='tos', mip='Omon', ) - ds.add_ancillary(short_name='areacello', mip='Ofx') - ds.add_ancillary(short_name='volcello') + ds.add_supplementary(short_name='areacello', mip='Ofx') + ds.add_supplementary(short_name='volcello') expected = ("Dataset: tos, Omon, CMIP6, dataset1, " - "ancillaries: areacello, Ofx; volcello") + "supplementaries: areacello, Ofx; volcello") assert ds.summary(shorten=True) == expected @@ -75,14 +66,14 @@ def test_long_summary(): def test_session_setter(): ds = Dataset(short_name='tas') - ds.add_ancillary(short_name='areacella') + ds.add_supplementary(short_name='areacella') assert ds._session is None - assert ds.ancillaries[0]._session is None + assert ds.supplementaries[0]._session is None ds.session assert isinstance(ds.session, esmvalcore.config.Session) - assert ds.session == ds.ancillaries[0].session + assert ds.session == ds.supplementaries[0].session @pytest.mark.parametrize( @@ -216,6 +207,348 @@ def test_augment_facets(session, facets, added_facets): assert dataset.facets == expected_facets +def test_from_recipe(session, tmp_path): + recipe_txt = textwrap.dedent(""" + + diagnostics: + diagnostic1: + variables: + tas: + project: CMIP5 + mip: Amon + additional_datasets: + - {dataset: dataset1} + """) + recipe = tmp_path / 'recipe_test.yml' + recipe.write_text(recipe_txt, encoding='utf-8') + + dataset = Dataset( + diagnostic='diagnostic1', + variable_group='tas', + short_name='tas', + dataset='dataset1', + project='CMIP5', + mip='Amon', + alias='dataset1', + recipe_dataset_index=0, + ) + dataset.session = session + + print(Dataset.from_recipe(recipe, session)) + print([dataset]) + assert Dataset.from_recipe(recipe, session) == [dataset] + + +def test_from_recipe_advanced(session, tmp_path): + recipe_txt = textwrap.dedent(""" + + datasets: + - {dataset: 'dataset1', project: CMIP6} + + diagnostics: + diagnostic1: + additional_datasets: + - {dataset: 'dataset2', project: CMIP6} + variables: + ta: + mip: Amon + pr: + mip: Amon + additional_datasets: + - {dataset: 'dataset3', project: CMIP5} + diagnostic2: + variables: + tos: + mip: Omon + """) + recipe = tmp_path / 'recipe_test.yml' + recipe.write_text(recipe_txt, encoding='utf-8') + + datasets = [ + Dataset( + diagnostic='diagnostic1', + variable_group='ta', + short_name='ta', + dataset='dataset1', + project='CMIP6', + mip='Amon', + alias='CMIP6_dataset1', + recipe_dataset_index=0, + ), + Dataset( + diagnostic='diagnostic1', + variable_group='ta', + short_name='ta', + dataset='dataset2', + project='CMIP6', + mip='Amon', + alias='CMIP6_dataset2', + recipe_dataset_index=1, + ), + Dataset( + diagnostic='diagnostic1', + variable_group='pr', + short_name='pr', + dataset='dataset1', + project='CMIP6', + mip='Amon', + alias='CMIP6_dataset1', + recipe_dataset_index=0, + ), + Dataset( + diagnostic='diagnostic1', + variable_group='pr', + short_name='pr', + dataset='dataset2', + project='CMIP6', + mip='Amon', + alias='CMIP6_dataset2', + recipe_dataset_index=1, + ), + Dataset( + diagnostic='diagnostic1', + variable_group='pr', + short_name='pr', + dataset='dataset3', + project='CMIP5', + mip='Amon', + alias='CMIP5', + recipe_dataset_index=2, + ), + Dataset( + diagnostic='diagnostic2', + variable_group='tos', + short_name='tos', + dataset='dataset1', + project='CMIP6', + mip='Omon', + alias='dataset1', + recipe_dataset_index=0, + ), + ] + for dataset in datasets: + dataset.session = session + + assert Dataset.from_recipe(recipe, session) == datasets + + +def test_from_recipe_with_ranges(session, tmp_path): + recipe_txt = textwrap.dedent(""" + + datasets: + - {dataset: 'dataset1', ensemble: r(1:2)i1p1} + + diagnostics: + diagnostic1: + variables: + ta: + mip: Amon + project: CMIP6 + """) + recipe = tmp_path / 'recipe_test.yml' + recipe.write_text(recipe_txt, encoding='utf-8') + + datasets = [ + Dataset( + diagnostic='diagnostic1', + variable_group='ta', + short_name='ta', + dataset='dataset1', + ensemble='r1i1p1', + project='CMIP6', + mip='Amon', + alias='r1i1p1', + recipe_dataset_index=0, + ), + Dataset( + diagnostic='diagnostic1', + variable_group='ta', + short_name='ta', + dataset='dataset1', + ensemble='r2i1p1', + project='CMIP6', + mip='Amon', + alias='r2i1p1', + recipe_dataset_index=1, + ), + ] + for dataset in datasets: + dataset.session = session + + assert Dataset.from_recipe(recipe, session) == datasets + + +def test_from_recipe_with_supplementary(session, tmp_path): + recipe_txt = textwrap.dedent(""" + + datasets: + - {dataset: 'dataset1', ensemble: r1i1p1} + + diagnostics: + diagnostic1: + variables: + tos: + project: CMIP5 + mip: Omon + supplementary_variables: + - short_name: sftof + mip: fx + """) + recipe = tmp_path / 'recipe_test.yml' + recipe.write_text(recipe_txt, encoding='utf-8') + + dataset = Dataset( + diagnostic='diagnostic1', + variable_group='tos', + short_name='tos', + dataset='dataset1', + ensemble='r1i1p1', + project='CMIP5', + mip='Omon', + alias='dataset1', + recipe_dataset_index=0, + ) + dataset.supplementaries = [ + Dataset( + short_name='sftof', + dataset='dataset1', + ensemble='r1i1p1', + project='CMIP5', + mip='fx', + ), + ] + dataset.session = session + + assert Dataset.from_recipe(recipe, session) == [dataset] + + +def test_from_recipe_with_skip_supplementary(session, tmp_path): + session['use_legacy_supplementaries'] = False + + recipe_txt = textwrap.dedent(""" + + datasets: + - {dataset: 'dataset1', ensemble: r1i1p1} + + diagnostics: + diagnostic1: + variables: + tos: + project: CMIP5 + mip: Omon + supplementary_variables: + - short_name: sftof + mip: fx + - short_name: areacello + skip: true + """) + recipe = tmp_path / 'recipe_test.yml' + recipe.write_text(recipe_txt, encoding='utf-8') + + dataset = Dataset( + diagnostic='diagnostic1', + variable_group='tos', + short_name='tos', + dataset='dataset1', + ensemble='r1i1p1', + project='CMIP5', + mip='Omon', + alias='dataset1', + recipe_dataset_index=0, + ) + dataset.supplementaries = [ + Dataset( + short_name='sftof', + dataset='dataset1', + ensemble='r1i1p1', + project='CMIP5', + mip='fx', + ), + ] + dataset.session = session + + assert Dataset.from_recipe(recipe, session) == [dataset] + + +def test_from_recipe_with_automatic_supplementary(session, tmp_path, + monkeypatch): + session['use_legacy_supplementaries'] = False + + def _find_files(self): + if self.facets['short_name'] == 'areacello': + file = esmvalcore.local.LocalFile() + file.facets = { + 'short_name': 'areacello', + 'mip': 'fx', + 'project': 'CMIP5', + 'dataset': 'dataset1', + 'ensemble': 'r0i0p0', + 'exp': 'piControl', + 'institute': 'X', + 'product': 'output1', + 'version': 'v2', + } + files = [file] + else: + files = [] + self._files = files + + monkeypatch.setattr(Dataset, '_find_files', _find_files) + recipe_txt = textwrap.dedent(""" + + preprocessors: + global_mean: + area_statistics: + operator: mean + + datasets: + - {dataset: 'dataset1', ensemble: r1i1p1} + + diagnostics: + diagnostic1: + variables: + tos: + project: CMIP5 + mip: Omon + exp: historical + preprocessor: global_mean + version: v1 + """) + recipe = tmp_path / 'recipe_test.yml' + recipe.write_text(recipe_txt, encoding='utf-8') + + dataset = Dataset( + diagnostic='diagnostic1', + variable_group='tos', + short_name='tos', + dataset='dataset1', + ensemble='r1i1p1', + exp='historical', + preprocessor='global_mean', + project='CMIP5', + version='v1', + mip='Omon', + alias='dataset1', + recipe_dataset_index=0, + ) + dataset.supplementaries = [ + Dataset( + short_name='areacello', + dataset='dataset1', + institute='X', + product='output1', + ensemble='r0i0p0', + exp='piControl', + project='CMIP5', + version='v2', + mip='fx', + ), + ] + dataset.session = session + + assert Dataset.from_recipe(recipe, session) == [dataset] + + @pytest.mark.parametrize('pattern,result', ( ['a', False], ['*', True], @@ -226,7 +559,20 @@ def test_isglob(pattern, result): assert esmvalcore.dataset._isglob(pattern) == result -def test_from_files(session, mocker): +def mock_find_files(*files): + files_map = defaultdict(list) + for file in files: + files_map[file.facets['short_name']].append(file) + + def find_files(self): + self.files = files_map[self['short_name']] + for supplementary in self.supplementaries: + supplementary.files = files_map[supplementary['short_name']] + + return find_files + + +def test_from_files(session, monkeypatch): rootpath = Path('/path/to/data') file1 = esmvalcore.local.LocalFile( rootpath, @@ -293,6 +639,9 @@ def test_from_files(session, mocker): 'grid': 'gn', 'version': 'v20190815', } + find_files = mock_find_files(file1, file2, file3) + monkeypatch.setattr(Dataset, 'find_files', find_files) + dataset = Dataset( short_name='tas', mip='Amon', @@ -300,13 +649,8 @@ def test_from_files(session, mocker): dataset='*', ) dataset.session = session - mocker.patch.object( - Dataset, - 'files', - new_callable=mocker.PropertyMock, - side_effect=[[file1, file2, file3]], - ) datasets = list(dataset.from_files()) + expected = [ Dataset(short_name='tas', mip='Amon', @@ -324,8 +668,33 @@ def test_from_files(session, mocker): assert datasets == expected -def test_from_files_with_ancillary(session, mocker): +def test_from_files_with_supplementary(session, monkeypatch): rootpath = Path('/path/to/data') + file = esmvalcore.local.LocalFile( + rootpath, + 'CMIP6', + 'CMIP', + 'CAS', + 'FGOALS-g3', + 'historical', + 'r3i1p1f1', + 'Amon', + 'tas', + 'gn', + 'v20190827', + 'tas_Amon_FGOALS-g3_historical_r3i1p1f1_gn_199001-199912.nc', + ) + file.facets = { + 'activity': 'CMIP', + 'institute': 'CAS', + 'dataset': 'FGOALS-g3', + 'exp': 'historical', + 'mip': 'Amon', + 'ensemble': 'r3i1p1f1', + 'short_name': 'tas', + 'grid': 'gn', + 'version': 'v20190827', + } afile = esmvalcore.local.LocalFile( rootpath, 'CMIP6', @@ -351,21 +720,17 @@ def test_from_files_with_ancillary(session, mocker): 'grid': 'gn', 'version': 'v20210615', } + monkeypatch.setattr(Dataset, 'find_files', mock_find_files(file, afile)) + dataset = Dataset( short_name='tas', mip='Amon', project='CMIP6', dataset='FGOALS-g3', - ensemble='r3i1p1f1', + ensemble='*', ) dataset.session = session - dataset.add_ancillary(short_name='areacella', mip='fx', ensemble='*') - mocker.patch.object( - Dataset, - 'files', - new_callable=mocker.PropertyMock, - side_effect=[[afile]], - ) + dataset.add_supplementary(short_name='areacella', mip='*', ensemble='*') expected = Dataset( short_name='tas', @@ -375,7 +740,7 @@ def test_from_files_with_ancillary(session, mocker): ensemble='r3i1p1f1', ) expected.session = session - expected.add_ancillary( + expected.add_supplementary( short_name='areacella', mip='fx', ensemble='r1i1p1f1', @@ -385,12 +750,12 @@ def test_from_files_with_ancillary(session, mocker): assert all(ds.session == session for ds in datasets) assert all(ads.session == session for ds in datasets - for ads in ds.ancillaries) + for ads in ds.supplementaries) assert datasets == [expected] -def test_from_files_with_globs(mocker, session): - """Test `from_files` with wildcards in dataset and ancillary.""" +def test_from_files_with_globs(monkeypatch, session): + """Test `from_files` with wildcards in dataset and supplementary.""" rootpath = Path('/path/to/data') file = esmvalcore.local.LocalFile( rootpath, @@ -455,7 +820,7 @@ def test_from_files_with_globs(mocker, session): project='CMIP6', short_name='tas', ) - dataset.add_ancillary( + dataset.add_supplementary( short_name='areacella', mip='fx', activity='*', @@ -465,20 +830,13 @@ def test_from_files_with_globs(mocker, session): dataset.session = session print(dataset) - mocker.patch.object( - Dataset, - 'files', - new_callable=mocker.PropertyMock, - # One call to `files` in `_get_available_facets` for 'tas' - # Two calls to `files` in `_update_timerange` for 'tas' - # One call to `files` in `_get_available facets` for 'areacella' - side_effect=[[file], [file], [file], [afile]], - ) + monkeypatch.setattr(Dataset, 'find_files', mock_find_files(file, afile)) + datasets = list(dataset.from_files()) assert all(ds.session == session for ds in datasets) assert all(ads.session == session for ds in datasets - for ads in ds.ancillaries) + for ads in ds.supplementaries) expected = Dataset( activity='CMIP', @@ -491,7 +849,7 @@ def test_from_files_with_globs(mocker, session): project='CMIP6', short_name='tas', ) - expected.add_ancillary( + expected.add_supplementary( short_name='areacella', mip='fx', activity='GMMIP', @@ -503,6 +861,348 @@ def test_from_files_with_globs(mocker, session): assert datasets == [expected] +def test_from_files_with_globs_and_missing_facets(monkeypatch, session): + """Test `from_files` with wildcards and files with missing facets. + + Tests a combination of files with complete facets and missing facets. + """ + rootpath = Path('/path/to/data') + file1 = esmvalcore.local.LocalFile( + rootpath, + 'CMIP6', + 'CMIP', + 'BCC', + 'BCC-CSM2-MR', + 'historical', + 'r1i1p1f1', + 'Amon', + 'tas', + 'gn', + 'v20181126', + 'tas_Amon_BCC-CSM2-MR_historical_r1i1p1f1_gn_185001-201412.nc', + ) + file1.facets = { + 'activity': 'CMIP', + 'dataset': 'BCC-CSM2-MR', + 'exp': 'historical', + 'ensemble': 'r1i1p1f1', + 'grid': 'gn', + 'institute': 'BCC', + 'mip': 'Amon', + 'project': 'CMIP6', + 'short_name': 'tas', + 'version': 'v20181126', + } + file2 = esmvalcore.local.LocalFile( + rootpath, + 'tas', + 'tas_Amon_BCC-CSM2-MR_historical_r1i1p1f1_gn_185001-201412.nc', + ) + file2.facets = { + 'short_name': 'tas', + } + + dataset = Dataset( + activity='CMIP', + dataset='*', + ensemble='r1i1p1f1', + exp='historical', + grid='gn', + institute='*', + mip='Amon', + project='CMIP6', + short_name='tas', + timerange='*', + ) + + dataset.session = session + print(dataset) + + monkeypatch.setattr(Dataset, 'find_files', mock_find_files(file1, file2)) + + datasets = list(dataset.from_files()) + + assert all(ds.session == session for ds in datasets) + assert all(ads.session == session for ds in datasets + for ads in ds.supplementaries) + + expected = Dataset( + activity='CMIP', + dataset='BCC-CSM2-MR', + ensemble='r1i1p1f1', + exp='historical', + grid='gn', + institute='BCC', + mip='Amon', + project='CMIP6', + short_name='tas', + timerange='185001/201412', + ) + + expected.session = session + + assert datasets == [expected] + + +def test_from_files_with_globs_and_automatic_missing(monkeypatch, session): + """Test `from_files`. + + Test with wildcards and files with missing facets that can be automatically + added. + """ + rootpath = Path('/path/to/data') + file = esmvalcore.local.LocalFile( + rootpath, + 'CMIP6', + 'BCC-CSM2-MR', + 'historical', + 'r1i1p1f1', + 'Amon', + 'tas', + 'gn', + 'v20181126', + 'tas_Amon_BCC-CSM2-MR_historical_r1i1p1f1_gn_185001-201412.nc', + ) + file.facets = { + 'dataset': 'BCC-CSM2-MR', + 'exp': 'historical', + 'ensemble': 'r1i1p1f1', + 'grid': 'gn', + 'mip': 'Amon', + 'project': 'CMIP6', + 'short_name': 'tas', + 'version': 'v20181126', + } + + dataset = Dataset( + activity='CMIP', + dataset='*', + ensemble='r1i1p1f1', + exp='historical', + grid='gn', + institute='*', + mip='Amon', + project='CMIP6', + short_name='tas', + timerange='*', + ) + + dataset.session = session + print(dataset) + + monkeypatch.setattr(Dataset, 'find_files', mock_find_files(file)) + + datasets = list(dataset.from_files()) + + assert all(ds.session == session for ds in datasets) + assert all(ads.session == session for ds in datasets + for ads in ds.supplementaries) + + expected = Dataset( + activity='CMIP', + dataset='BCC-CSM2-MR', + ensemble='r1i1p1f1', + exp='historical', + grid='gn', + mip='Amon', + project='CMIP6', + short_name='tas', + timerange='185001/201412', + ) + + expected.session = session + + assert datasets == [expected] + + +def test_from_files_with_globs_and_only_missing_facets(monkeypatch, session): + """Test `from_files` with wildcards and only files with missing facets.""" + rootpath = Path('/path/to/data') + file = esmvalcore.local.LocalFile( + rootpath, + 'CMIP6', + 'CMIP', + 'BCC', + 'BCC-CSM2-MR', + 'historical', + 'r1i1p1f1', + 'Amon', + 'tas', + 'v20181126', + 'tas_Amon_BCC-CSM2-MR_historical_r1i1p1f1_gn_185001-201412.nc', + ) + file.facets = { + 'activity': 'CMIP', + 'dataset': 'BCC-CSM2-MR', + 'exp': 'historical', + 'ensemble': 'r1i1p1f1', + 'institute': 'BCC', + 'mip': 'Amon', + 'project': 'CMIP6', + 'short_name': 'tas', + 'version': 'v20181126', + } + + dataset = Dataset( + activity='CMIP', + dataset='*', + ensemble='r1i1p1f1', + exp='historical', + grid='*', + institute='*', + mip='Amon', + project='CMIP6', + short_name='tas', + timerange='*', + ) + + dataset.session = session + print(dataset) + + monkeypatch.setattr(Dataset, 'find_files', mock_find_files(file)) + + datasets = list(dataset.from_files()) + + assert all(ds.session == session for ds in datasets) + assert all(ads.session == session for ds in datasets + for ads in ds.supplementaries) + + expected = Dataset( + activity='CMIP', + dataset='BCC-CSM2-MR', + ensemble='r1i1p1f1', + exp='historical', + grid='*', + institute='BCC', + mip='Amon', + project='CMIP6', + short_name='tas', + timerange='*', + ) + + expected.session = session + + assert datasets == [expected] + + +def test_match(): + dataset1 = Dataset( + short_name='areacella', + ensemble=['r1i1p1f1'], + exp='historical', + modeling_realm=['atmos', 'land'], + ) + dataset2 = Dataset( + short_name='tas', + ensemble='r1i1p1f1', + exp=['historical', 'ssp585'], + modeling_realm=['atmos'], + ) + + score = dataset1._match(dataset2) + assert score == 3 + + +def test_remove_duplicate_supplementaries(): + dataset = Dataset( + dataset='dataset1', + short_name='tas', + mip='Amon', + project='CMIP6', + exp='historical', + ) + supplementary1 = dataset.copy(short_name='areacella') + supplementary2 = supplementary1.copy() + supplementary1.facets['exp'] = '1pctCO2' + dataset.supplementaries = [supplementary1, supplementary2] + + dataset._remove_duplicate_supplementaries() + + assert len(dataset.supplementaries) == 1 + assert dataset.supplementaries[0] == supplementary2 + + +def test_remove_not_found_supplementaries(): + dataset = Dataset( + dataset='dataset1', + short_name='tas', + mip='Amon', + project='CMIP6', + exp='historical', + ) + dataset.add_supplementary(short_name='areacella', mip='fx', exp='*') + dataset._remove_unexpanded_supplementaries() + + assert len(dataset.supplementaries) == 0 + + +def test_from_recipe_with_glob(tmp_path, session, mocker): + recipe_txt = textwrap.dedent(""" + + diagnostics: + diagnostic1: + variables: + tas: + project: CMIP5 + mip: Amon + additional_datasets: + - {dataset: '*'} + """) + recipe = tmp_path / 'recipe_test.yml' + recipe.write_text(recipe_txt, encoding='utf-8') + + session['drs']['CMIP5'] = 'ESGF' + + filenames = [ + "cmip5/output1/CSIRO-QCCCE/CSIRO-Mk3-6-0/rcp85/mon/atmos/Amon/r4i1p1/" + "v20120323/tas_Amon_CSIRO-Mk3-6-0_rcp85_r4i1p1_200601-210012.nc", + "cmip5/output1/NIMR-KMA/HadGEM2-AO/historical/mon/atmos/Amon/r1i1p1/" + "v20130815/tas_Amon_HadGEM2-AO_historical_r1i1p1_186001-200512.nc", + ] + + mocker.patch.object( + esmvalcore.local, + '_get_input_filelist', + autospec=True, + spec_set=True, + return_value=(filenames, []), + ) + + definitions = [ + { + 'diagnostic': 'diagnostic1', + 'variable_group': 'tas', + 'dataset': 'CSIRO-Mk3-6-0', + 'project': 'CMIP5', + 'mip': 'Amon', + 'short_name': 'tas', + 'alias': 'CSIRO-Mk3-6-0', + 'recipe_dataset_index': 0, + }, + { + 'diagnostic': 'diagnostic1', + 'variable_group': 'tas', + 'dataset': 'HadGEM2-AO', + 'project': 'CMIP5', + 'mip': 'Amon', + 'short_name': 'tas', + 'alias': 'HadGEM2-AO', + 'recipe_dataset_index': 1, + }, + ] + expected = [] + for facets in definitions: + dataset = Dataset(**facets) + dataset.session = session + expected.append(dataset) + + datasets = Dataset.from_recipe(recipe, session) + print("Expected:", expected) + print("Got:", datasets) + assert all(ds.session == session for ds in datasets) + assert datasets == expected + + def test_from_ranges(): dataset = Dataset(ensemble='r(1:2)i1p1f1') expected = [ @@ -737,7 +1437,7 @@ def test_find_files_outdated_local(mocker, dataset): def test_set_version(): dataset = Dataset(short_name='tas') - dataset.add_ancillary(short_name='areacella') + dataset.add_supplementary(short_name='areacella') file_v1 = esmvalcore.local.LocalFile('/path/to/v1/tas.nc') file_v1.facets['version'] = 'v1' file_v2 = esmvalcore.local.LocalFile('/path/to/v2/tas.nc') @@ -745,10 +1445,10 @@ def test_set_version(): afile = esmvalcore.local.LocalFile('/path/to/v3/areacella.nc') afile.facets['version'] = 'v3' dataset.files = [file_v2, file_v1] - dataset.ancillaries[0].files = [afile] + dataset.supplementaries[0].files = [afile] dataset.set_version() assert dataset.facets['version'] == ['v1', 'v2'] - assert dataset.ancillaries[0].facets['version'] == 'v3' + assert dataset.supplementaries[0].facets['version'] == 'v3' @pytest.mark.parametrize('timerange', ['*', '185001/*', '*/185112']) @@ -822,13 +1522,19 @@ def test_update_timerange_no_files(session, offline): } dataset = Dataset(**variable) dataset.files = [] - msg = r"Missing data for CMIP6: tas.*" + msg = r"Missing data for Dataset: tas, Amon, CMIP6, HadGEM3-GC31-LL.*" with pytest.raises(InputFilesNotFound, match=msg): dataset._update_timerange() def test_update_timerange_typeerror(): - dataset = Dataset(timerange=42) + dataset = Dataset( + short_name='tas', + mip='Amon', + project='CMIP6', + dataset='dataset1', + timerange=42, + ) msg = r"timerange should be a string, got '42'" with pytest.raises(TypeError, match=msg): dataset._update_timerange() @@ -857,7 +1563,8 @@ def test_load(mocker, session): args = {} order = [] - def mock_preprocess(items, step, input_files, **kwargs): + def mock_preprocess(items, step, input_files, output_file, debug, + **kwargs): order.append(step) args[step] = kwargs return items @@ -880,12 +1587,14 @@ def mock_preprocess(items, step, input_files, **kwargs): 'clip_timerange', 'fix_data', 'cmor_check_data', - 'add_ancillary_variables', + 'add_supplementary_variables', ] assert order == load_order load_args = { - 'load': {}, + 'load': { + 'callback': 'default' + }, 'fix_file': { 'dataset': 'CanESM2', 'ensemble': 'r1i1p1', @@ -937,8 +1646,8 @@ def mock_preprocess(items, step, input_files, **kwargs): 'frequency': 'yr', }, 'concatenate': {}, - 'add_ancillary_variables': { - 'ancillary_cubes': [], + 'add_supplementary_variables': { + 'supplementary_cubes': [], }, }