Support multiple paths in input_dir (#1000)

* Support multiple folder conventions in config-developer

* Add documentation

* Fix tests

doc/quickstart/configure.rst

@@ -220,7 +220,7 @@ Input file paths
 ----------------
 
 When looking for input files, the ``esmvaltool`` command provided by
-ESMValCore replaces the placeholders ``[item]`` in
+ESMValCore replaces the placeholders ``{item}`` in
 ``input_dir`` and ``input_file`` with the values supplied in the recipe.
 ESMValCore will try to automatically fill in the values for institute, frequency,
 and modeling_realm based on the information provided in the CMOR tables
@@ -240,6 +240,21 @@ The resulting directory path would look something like this:
 
     CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f3/Omon/tos/gn/latest
 
+Please, bear in mind that ``input_dirs`` can also be a list for those  cases in
+which may be needed:
+
+.. code-block:: yaml
+
+  - '{exp}/{ensemble}/original/{mip}/{short_name}/{grid}/{latestversion}'
+  - '{exp}/{ensemble}/computed/{mip}/{short_name}/{grid}/{latestversion}'
+
+In that case, the resultant directories will be:
+
+.. code-block:: bash
+
+  historical/r1i1p1f3/original/Omon/tos/gn/latest
+  historical/r1i1p1f3/computed/Omon/tos/gn/latest
+
 For a more in-depth description of how to configure ESMValCore so it can find
 your data please see :ref:`CMOR-DRS`.
 

esmvalcore/_data_finder.py

@@ -89,11 +89,18 @@ def select_files(filenames, start_year, end_year):
     return selection
 
 
-def _replace_tags(path, variable):
+def _replace_tags(paths, variable):
     """Replace tags in the config-developer's file with actual values."""
-    path = path.strip('/')
-    tlist = re.findall(r'{([^}]*)}', path)
-    paths = [path]
+    if isinstance(paths, str):
+        paths = (paths.strip('/'), )
+    else:
+        paths = [path.strip('/') for path in paths]
+    tlist = set()
+
+    for path in paths:
+        tlist = tlist.union(re.findall(r'{([^}]*)}', path))
+    logger.debug(tlist)
+
     for tag in tlist:
         original_tag = tag
         tag, _, _ = _get_caps_options(tag)

tests/unit/data_finder/test_replace_tags.py

@@ -0,0 +1,22 @@
+"""Tests for _replace_tags in _data_finder.py."""
+
+from esmvalcore._data_finder import _replace_tags
+
+VARIABLE = {
+    'short_name': 'tas',
+}
+
+
+def test_replace_tags_str():
+    assert _replace_tags('folder/subfolder/{short_name}',
+                         VARIABLE) == ['folder/subfolder/tas']
+
+
+def test_replace_tags_list_of_str():
+    assert _replace_tags(('folder/subfolder/{short_name}',
+                          'folder2/{short_name}', 'subfolder/{short_name}'),
+                         VARIABLE) == [
+                             'folder/subfolder/tas',
+                             'folder2/tas',
+                             'subfolder/tas',
+                         ]