Increase test coverage

Author: ghiggi

disdrodb/api/create_directories.py

@@ -415,7 +415,6 @@ def create_initial_station_structure(
         data_source=data_source,
         campaign_name=campaign_name,
         station_name=station_name,
-        product="RAW",
     )
     # Add default station issue file
     create_station_issue(
@@ -440,7 +439,7 @@ def create_test_archive(test_base_dir, data_source, campaign_name, station_name,
     This enable to then test data download and DISDRODB processing.
     """
     # Check test_base_dir is not equal to true base_dir
-    if test_base_dir == get_base_dir():
+    if test_base_dir == get_base_dir(base_dir):
         raise ValueError("Provide a test_base_dir directory different from the true DISDRODB base directory !")
 
     # Create test DISDRODB base directory

disdrodb/data_transfer/scripts/disdrodb_upload_station.py

@@ -22,8 +22,8 @@
 
 import click
 
-from disdrodb.data_transfer.upload_data import click_base_dir_option, click_upload_options
-from disdrodb.utils.scripts import click_station_arguments
+from disdrodb.data_transfer.upload_data import click_upload_options
+from disdrodb.utils.scripts import click_base_dir_option, click_station_arguments
 
 sys.tracebacklimit = 0  # avoid full traceback error if occur
 

disdrodb/data_transfer/upload_data.py

@@ -111,6 +111,13 @@ def _filter_already_uploaded(metadata_filepaths: List[str], force: bool) -> List
     return filtered
 
 
+def _check_valid_platform(platform):
+    """Check upload platform validity."""
+    valid_platform = ["zenodo", "sandbox.zenodo"]
+    if platform not in valid_platform:
+        raise NotImplementedError(f"Invalid platform {platform}. Valid platforms are {valid_platform}.")
+
+
 def upload_station(
     data_source: str,
     campaign_name: str,
@@ -145,6 +152,8 @@ def upload_station(
         The default is force=False.
 
     """
+    _check_valid_platform(platform)
+
     # Define metadata_filepath
     metadata_filepath = define_metadata_filepath(
         data_source=data_source,
@@ -162,11 +171,8 @@ def upload_station(
     if platform == "zenodo":
         upload_station_to_zenodo(metadata_filepath, sandbox=False)
 
-    elif platform == "sandbox.zenodo":  # Only for testing purposes, not available through CLI
+    else:  # platform == "sandbox.zenodo":  # Only for testing purposes, not available through CLI
         upload_station_to_zenodo(metadata_filepath, sandbox=True)
-    else:
-        valid_platform = ["zenodo", "sandbox.zenodo"]
-        raise NotImplementedError(f"Invalid platform {platform}. Valid platforms are {valid_platform}.")
 
 
 def upload_archive(
@@ -206,6 +212,8 @@ def upload_archive(
         If not provided (None), all stations will be uploaded.
         The default is station_name=None.
     """
+    _check_valid_platform(platform)
+
     # Get list metadata
     metadata_filepaths = get_list_metadata(
         **kwargs,

disdrodb/data_transfer/zenodo.py

@@ -151,11 +151,20 @@ def _define_creators_list(metadata):
         list_names = re.split(";|,", metadata["authors"])
         list_identifier = re.split(";|,", metadata["authors_url"])
         list_affiliation = re.split(";|,", metadata["institution"])
+
+        # Check identifier fields match the number of specified authors
+        # - If not, set identifier to ""
         if len(list_names) != len(list_identifier):
-            # print("Impossible to automatically assign identifier to authors. Length mismatch.")
             list_identifier = [""] * len(list_names)
+
+        # If affiliation is only one --> Assume is the same for everyone
         if len(list_affiliation) == 1:
             list_affiliation = list_affiliation * len(list_names)
+        # If more than one affiliation, but does not match list of authors, set to ""
+        if len(list_affiliation) > 1 and len(list_affiliation) != len(list_names):
+            list_affiliation = [""] * len(list_names)
+
+        # Create info dictionary of each author
         list_creator_dict = []
         for name, identifier, affiliation in zip(list_names, list_identifier, list_affiliation):
             creator_dict = {}
@@ -224,7 +233,7 @@ def upload_station_to_zenodo(metadata_filepath: str, sandbox: bool = True) -> st
         os.remove(station_zip_filepath)
     except Exception as e:
         os.remove(station_zip_filepath)
-        raise ValueError(f"The upload on Zenodo has failed: {e}.")
+        raise ValueError(f"{station_zip_filepath} The upload on Zenodo has failed: {e}.")
 
     # Add the disdrodb_data_url information to the metadata
     print(" - The station metadata 'disdrodb_data_url' key has been updated with the remote url")

disdrodb/issue/writer.py

@@ -50,7 +50,7 @@ def _write_issue_docs(f):
     return None
 
 
-def _write_issue(filepath: str, timesteps: list = None, time_periods: list = None) -> None:
+def write_issue(filepath: str, timesteps: list = None, time_periods: list = None) -> None:
     """Write the issue YAML file.
 
     Parameters
@@ -92,35 +92,6 @@ def _write_issue(filepath: str, timesteps: list = None, time_periods: list = Non
     return None
 
 
-def write_issue_dict(filepath: str, issue_dict: dict) -> None:
-    """Write the issue YAML file.
-
-    Parameters
-    ----------
-    filepath : str
-        Filepath of the issue YAML to write.
-    issue_dict : dict
-        Issue dictionary
-    """
-    _write_issue(
-        filepath=filepath,
-        timesteps=issue_dict.get("timesteps", None),
-        time_periods=issue_dict.get("time_periods", None),
-    )
-
-
-def write_default_issue(filepath: str) -> None:
-    """Write an empty issue YAML file.
-
-    Parameters
-    ----------
-    filepath : str
-        Filepath of the issue YAML to write.
-    """
-    _write_issue(filepath=filepath)
-    return None
-
-
 def create_station_issue(data_source, campaign_name, station_name, base_dir=None):
     """Write an empty YAML issue YAML file for a DISDRODB station.
 
@@ -155,6 +126,6 @@ def create_station_issue(data_source, campaign_name, station_name, base_dir=None
     issue_dir = os.path.dirname(issue_filepath)
     os.makedirs(issue_dir, exist_ok=True)
     # Write issue file
-    write_default_issue(filepath=issue_filepath)
+    write_issue(filepath=issue_filepath)
     print(f"An empty issue YAML file for station {station_name} has been created .")
     return None

disdrodb/l0/l0_processing.py

@@ -22,6 +22,7 @@
 import functools
 import logging
 import os
+import shutil
 import time
 
 import dask
@@ -39,6 +40,7 @@
     define_l0a_filepath,
     define_l0b_filepath,
     define_l0b_station_dir,
+    define_station_dir,
     get_disdrodb_path,
 )
 from disdrodb.configs import get_base_dir
@@ -785,7 +787,7 @@ def run_l0b_from_nc(
     return None
 
 
-def run_l0b_concat(processed_dir, station_name, remove=False, verbose=False):
+def run_l0b_concat(processed_dir, station_name, verbose=False):
     """Concatenate all L0B netCDF files into a single netCDF file.
 
     The single netCDF file is saved at <processed_dir>/L0B.
@@ -836,13 +838,6 @@ def run_l0b_concat(processed_dir, station_name, remove=False, verbose=False):
     ds.close()
     del ds
 
-    # -------------------------------------------------------------------------.
-    # If remove = True, remove all the single files
-    if remove:
-        log_info(logger=logger, msg="Removal of single L0B files started.", verbose=verbose)
-        _ = [os.remove(filepath) for filepath in filepaths]
-        log_info(logger=logger, msg="Removal of single L0B files ended.", verbose=verbose)
-
     # -------------------------------------------------------------------------.
     # Close the file logger
     close_logger(logger)
@@ -948,6 +943,7 @@ def run_l0b_station(
     verbose: bool = True,
     parallel: bool = True,
     debugging_mode: bool = False,
+    remove_l0a: bool = False,
     base_dir: str = None,
 ):
     """
@@ -1005,6 +1001,18 @@ def run_l0b_station(
         parallel=parallel,
     )
 
+    if remove_l0a:
+        station_dir = define_station_dir(
+            base_dir=base_dir,
+            product="L0A",
+            data_source=data_source,
+            campaign_name=campaign_name,
+            station_name=station_name,
+        )
+        log_info(logger=logger, msg="Removal of single L0A files started.", verbose=verbose)
+        shutil.rmtree(station_dir)
+        log_info(logger=logger, msg="Removal of single L0A files ended.", verbose=verbose)
+
 
 def run_l0b_concat_station(
     # Station arguments
@@ -1053,9 +1061,20 @@ def run_l0b_concat_station(
     run_l0b_concat(
         processed_dir=processed_dir,
         station_name=station_name,
-        remove=remove_l0b,
         verbose=verbose,
     )
 
+    if remove_l0b:
+        station_dir = define_station_dir(
+            base_dir=base_dir,
+            product="L0B",
+            data_source=data_source,
+            campaign_name=campaign_name,
+            station_name=station_name,
+        )
+        log_info(logger=logger, msg="Removal of single L0B files started.", verbose=verbose)
+        shutil.rmtree(station_dir)
+        log_info(logger=logger, msg="Removal of single L0B files ended.", verbose=verbose)
+
 
 ####---------------------------------------------------------------------------.

disdrodb/l0/routines.py

@@ -20,8 +20,6 @@
 
 import datetime
 import logging
-import os
-import shutil
 import time
 
 import click
@@ -107,6 +105,17 @@ def click_l0_processing_options(function: object):
     return function
 
 
+def click_remove_l0a_option(function: object):
+    function = click.option(
+        "--remove_l0a",
+        type=bool,
+        show_default=True,
+        default=False,
+        help="If true, remove the L0A files once the L0B processing is terminated.",
+    )(function)
+    return function
+
+
 def click_l0_archive_options(function: object):
     """Click command line arguments for L0 processing archiving of a station.
 
@@ -228,6 +237,7 @@ def run_disdrodb_l0b_station(
     debugging_mode: bool = False,
     parallel: bool = True,
     base_dir: str = None,
+    remove_l0a: bool = False,
 ):
     """Run the L0B processing of a station calling disdrodb_run_l0b_station in the terminal."""
     # Define command
@@ -247,6 +257,8 @@ def run_disdrodb_l0b_station(
             str(debugging_mode),
             "--parallel",
             str(parallel),
+            "--remove_l0a",
+            str(remove_l0a),
             "--base_dir",
             str(base_dir),
         ]
@@ -358,7 +370,6 @@ def run_disdrodb_l0_station(
         Base directory of DISDRODB. Format: <...>/DISDRODB
         If None (the default), the disdrodb config variable 'dir' is used.
     """
-    from disdrodb.api.path import get_disdrodb_path
 
     # ---------------------------------------------------------------------.
     t_i = time.time()
@@ -394,20 +405,9 @@ def run_disdrodb_l0_station(
             verbose=verbose,
             debugging_mode=debugging_mode,
             parallel=parallel,
+            remove_l0a=remove_l0a,
         )
 
-    # ------------------------------------------------------------------------.
-    # Remove L0A station directory if remove_l0a = True and l0b_processing = True
-    if l0b_processing and remove_l0a:
-        campaign_dir = get_disdrodb_path(
-            base_dir=base_dir,
-            product="L0A",
-            data_source=data_source,
-            campaign_name=campaign_name,
-        )
-        station_product_dir = os.path.join(campaign_dir, "L0A", station_name)
-        shutil.rmtree(station_product_dir)
-
     # ------------------------------------------------------------------------.
     # If l0b_concat=True, concat the netCDF in a single file
     if l0b_concat:
@@ -656,6 +656,7 @@ def run_disdrodb_l0b(
     debugging_mode: bool = False,
     parallel: bool = True,
     base_dir: str = None,
+    remove_l0a: bool = False,
 ):
     run_disdrodb_l0(
         base_dir=base_dir,
@@ -666,7 +667,7 @@ def run_disdrodb_l0b(
         l0a_processing=False,
         l0b_processing=True,
         l0b_concat=False,
-        remove_l0a=False,
+        remove_l0a=remove_l0a,
         remove_l0b=False,
         # Processing options
         force=force,
@@ -676,6 +677,7 @@ def run_disdrodb_l0b(
     )
 
 
+####---------------------------------------------------------------------------.
 def run_disdrodb_l0b_concat(
     data_sources=None,
     campaign_names=None,

disdrodb/l0/scripts/disdrodb_run_l0b.py

@@ -22,6 +22,7 @@
 from disdrodb.l0.routines import (
     click_l0_processing_options,
     click_l0_stations_options,
+    click_remove_l0a_option,
 )
 from disdrodb.utils.scripts import click_base_dir_option, parse_arg_to_list
 
@@ -31,6 +32,7 @@
 @click.command()
 @click_l0_stations_options
 @click_l0_processing_options
+@click_remove_l0a_option
 @click_base_dir_option
 def disdrodb_run_l0b(
     # L0 disdrodb stations options
@@ -42,6 +44,7 @@ def disdrodb_run_l0b(
     verbose: bool = True,
     parallel: bool = True,
     debugging_mode: bool = False,
+    remove_l0a: bool = False,
     base_dir: str = None,
 ):
     """
@@ -107,6 +110,7 @@ def disdrodb_run_l0b(
         verbose=verbose,
         debugging_mode=debugging_mode,
         parallel=parallel,
+        remove_l0a=remove_l0a,
     )
 
     return None