Merge branch 'fix_race_condition_cleanup' of github.com:ESMValGroup/ESMValCore into fix_race_condition_cleanup

Author: schlunma

esmvalcore/_recipe/to_datasets.py

@@ -361,9 +361,12 @@ def _get_datasets_for_variable(
                 template1.add_supplementary(**supplementary_facets)
             for supplementary_ds in template1.supplementaries:
                 supplementary_ds.facets.pop('preprocessor', None)
+                supplementary_ds['diagnostic'] = diagnostic_name
+                supplementary_ds['variable_group'] = variable_group
+                supplementary_ds['variable_group_subdir'] = 'supplementaries'
             for dataset in _dataset_from_files(template1):
-                dataset['variable_group'] = variable_group
                 dataset['diagnostic'] = diagnostic_name
+                dataset['variable_group'] = variable_group
                 dataset['recipe_dataset_index'] = idx  # type: ignore
                 logger.debug("Found %s", dataset.summary(shorten=True))
                 datasets.append(dataset)

esmvalcore/local.py

@@ -486,6 +486,7 @@ def _get_output_file(variable: dict[str, Any], preproc_dir: Path) -> Path:
         preproc_dir,
         variable.get('diagnostic', ''),
         variable.get('variable_group', ''),
+        variable.get('variable_group_subdir', ''),
         outfile,
     )
 

tests/integration/data_finder.yml

@@ -25,6 +25,24 @@ get_output_file:
     preproc_dir: /test
     output_file: /test/test_diag/test/CMIP5_HadGEM2-ES_Amon_historical-rcp85_r1i1p1_ta_1960-1980.nc
 
+  - variable:
+      variable_group: test
+      variable_group_subdir: supplementaries
+      short_name: areacella
+      original_short_name: areacella
+      dataset: HadGEM2-ES
+      project: CMIP5
+      institute: [INPE, MOHC]
+      frequency: fx
+      modeling_realm: [atmos]
+      mip: fx
+      exp: historical
+      ensemble: r0i0p0
+      diagnostic: test_diag
+      preprocessor: test_preproc
+    preproc_dir: /test
+    output_file: /test/test_diag/test/supplementaries/CMIP5_HadGEM2-ES_fx_historical_r0i0p0_areacella.nc
+
   # EMAC
 
   - variable:

tests/unit/recipe/test_to_datasets.py

@@ -178,6 +178,40 @@ def test_from_recipe_dict(session):
     assert len(datasets) == 1
 
 
+def test_supplementaries_have_subdir(session):
+    recipe_txt = textwrap.dedent("""
+    datasets:
+      - dataset: AWI-ESM-1-1-LR
+        grid: gn
+        supplementary_variables:
+          - short_name: sftof
+      - dataset: BCC-ESM1
+        grid: gn
+        supplementary_variables:
+          - short_name: sftlf
+            exp: 1pctCO2
+
+    diagnostics:
+      diagnostic1:
+        variables:
+          tas:
+            ensemble: r1i1p1f1
+            exp: historical
+            mip: Amon
+            project: CMIP6
+            supplementary_variables:
+              - short_name: areacella
+                mip: fx
+
+    """)
+    datasets = Dataset.from_recipe(recipe_txt, session)
+    print(datasets)
+    for dataset in datasets:
+        assert 'variable_group_subdir' not in dataset.facets
+        for supp_ds in dataset.supplementaries:
+            assert supp_ds.facets['variable_group_subdir'] == 'supplementaries'
+
+
 def test_merge_supplementaries_dataset_takes_priority(session):
     recipe_txt = textwrap.dedent("""
     datasets:
@@ -186,8 +220,8 @@ def test_merge_supplementaries_dataset_takes_priority(session):
       - dataset: BCC-ESM1
         grid: gn
         supplementary_variables:
-            - short_name: areacella
-              exp: 1pctCO2
+          - short_name: areacella
+            exp: 1pctCO2
 
     preprocessors:
       global_mean:
@@ -201,7 +235,7 @@ def test_merge_supplementaries_dataset_takes_priority(session):
             ensemble: r1i1p1f1
             exp: historical
             mip: Amon
-            preprocessor: global_mean_land
+            preprocessor: global_mean
             project: CMIP6
             supplementary_variables:
               - short_name: areacella

tests/unit/test_dataset.py

@@ -410,6 +410,9 @@ def test_from_recipe_with_supplementary(session, tmp_path):
     )
     dataset.supplementaries = [
         Dataset(
+            diagnostic='diagnostic1',
+            variable_group='tos',
+            variable_group_subdir='supplementaries',
             short_name='sftof',
             dataset='dataset1',
             ensemble='r1i1p1',
@@ -458,6 +461,9 @@ def test_from_recipe_with_skip_supplementary(session, tmp_path):
     )
     dataset.supplementaries = [
         Dataset(
+            diagnostic='diagnostic1',
+            variable_group='tos',
+            variable_group_subdir='supplementaries',
             short_name='sftof',
             dataset='dataset1',
             ensemble='r1i1p1',
@@ -533,6 +539,9 @@ def _find_files(self):
     )
     dataset.supplementaries = [
         Dataset(
+            diagnostic='diagnostic1',
+            variable_group='tos',
+            variable_group_subdir='supplementaries',
             short_name='areacello',
             dataset='dataset1',
             institute='X',