Fix tests

Author: ghiggi

docs/source/00_introduction.rst

@@ -32,7 +32,7 @@ The reference sensors for the GPM mission are the:
 The :ref:`GPM Data Archive <gpm_data_archive>` currently includes satellite data records that extend back to 1987.
 This extensive archive is the result of contributions from two spaceborne radars and a fleet of 26 passive microwave (PMW) sensors that forms the so-called GPM constellation.
 
-The data are organized into various :ref:` product levels <gpm_product_levels>`,
+The data are organized into various :ref:`product levels <gpm_product_levels>`,
 encompassing raw and calibrated observations (Level 1), intermediate geophysical retrieval products (Level 2),
 and spatio-temporally integrated datasets from single and multiple satellites (Level 3).
 
@@ -732,9 +732,9 @@ examination of land surface effects on radar, radiometer, or combined-sensor pre
 
 For detailed information on the TRMM/GPM-CloudSat Coincidence dataset, please
 refer to the corresponding `journal article <https://www.mdpi.com/2072-4292/13/12/2264>`_
-and the `Algorithm Theoretical Basis Document (ATBD) <https://gpm.nasa.gov/resources/documents/cloudsat-gpm-coincidence-dataset-version-1c>`_.
+and the `Algorithm Theoretical Basis Document (ATBD) <https://arthurhou.pps.eosdis.nasa.gov/Documents/CSAT_TRMM_GPM_COIN_ATBD_V05.pdf>`_.
 
-The coincidence dataset can be downloaded with GPM-API using the ``2B-GPM-CSAT`` and ``2B-TRMM-CSAT`` product acronyms.
+The TRMM/GPM-CloudSat Coincidence dataset V5 can be downloaded with GPM-API using the ``2B-GPM-CSAT`` and ``2B-TRMM-CSAT`` product acronyms.
 
 The figure below displays a quick-look radar time-height profile imagery from the GPM - CloudSat Coincidence dataset.
 

docs/source/03_quickstart.rst

@@ -133,7 +133,7 @@ specific time periods:
 .. code-block:: python
 
     gpm.available_products(end_time="1995-01-31")  # from the start of the mission to 1995-01-31
-    gpm.available_products(start_time="2014-01-01", end_time="2016", product_categories="PMW")
+    gpm.available_products(start_time="2014-01-01", end_time="2016-01-01", product_categories="PMW")
     gpm.available_products(start_time="2019-01-01")  # from 2019-01-01 to the present
 
 

gpm/configs.py

@@ -187,12 +187,14 @@ def _get_config_key(key):
 def get_base_dir(base_dir=None):
     """Return the GPM base directory."""
     import gpm
+    from gpm.io.checks import check_base_dir
 
     if base_dir is None:
         base_dir = gpm.config.get("base_dir")
     if base_dir is None:
         raise ValueError("The 'base_dir' is not specified in the GPM-API configuration file.")
-    return str(base_dir)  # convert Path to str
+    base_dir = check_base_dir(base_dir)  # convert Path to str
+    return base_dir
 
 
 def get_username_pps():

gpm/dataset/dataset.py

@@ -30,6 +30,7 @@
 
 import xarray as xr
 
+from gpm.configs import get_base_dir
 from gpm.dataset.conventions import finalize_dataset
 from gpm.dataset.granule import _multi_file_closer, get_scan_modes_datasets
 from gpm.io.checks import (
@@ -237,6 +238,7 @@ def open_dataset(
 
     """
     ## Check valid product and variables
+    base_dir = get_base_dir(base_dir=base_dir)
     product = check_product(product, product_type=product_type)
     variables = check_variables(variables)
     groups = check_groups(groups)
@@ -415,7 +417,7 @@ def open_datatree(
     product = check_product(product, product_type=product_type)
     variables = check_variables(variables)
     groups = check_groups(groups)
-
+    base_dir = get_base_dir(base_dir=base_dir)
     # Check scan_modes
     scan_modes = check_scan_modes(scan_modes=scan_modes, product=product, version=version)
 

gpm/dataset/decoding/coordinates.py

@@ -26,21 +26,31 @@
 # -----------------------------------------------------------------------------.
 """This module contains functions to sanitize GPM-API Dataset coordinates."""
 import numpy as np
+import xarray as xr
 
 
 def ensure_valid_coords(ds, raise_error=False):
     """Ensure geographic coordinates are within expected range."""
-    invalid_coords = np.logical_or(
-        np.logical_or(ds["lon"].data < -180, ds["lon"].data > 180),
-        np.logical_or(ds["lat"].data < -90, ds["lat"].data > 90),
+    ds["lon"] = ds["lon"].compute()
+    ds["lat"] = ds["lat"].compute()
+    da_invalid_coords = np.logical_or(
+        np.logical_or(ds["lon"] < -180, ds["lon"] > 180),
+        np.logical_or(ds["lat"] < -90, ds["lat"] > 90),
     )
-    if np.any(invalid_coords):
+    if np.any(da_invalid_coords.data):
         if raise_error:
             raise ValueError("Invalid geographic coordinate in the granule.")
-        da_invalid_coords = ds["lon"].copy()
-        da_invalid_coords.data = invalid_coords
-        # For each variable, set NaN value where invalid coordinates
-        ds = ds.where(~da_invalid_coords)
+
+        # # Add valid_geolocation flag
+        # ds = ds.assign_coords({"valid_geolocation": da_invalid_coords})
+
+        # # For each variable, set NaN value where invalid coordinates
+        # # --> Only if variable has at the 2 dimensions of ds["lon"]
+        # # --> Not good practice because it might mask quality flags DataArrays !
+        # for var in ds.data_vars:
+        #     if set(ds["lon"].dims).issubset(set(ds[var].dims)):
+        #         ds[var] = ds[var].where(~da_invalid_coords)
+
         # Add NaN to longitude and latitude
         ds["lon"] = ds["lon"].where(~da_invalid_coords)
         ds["lat"] = ds["lat"].where(~da_invalid_coords)
@@ -180,12 +190,37 @@ def _add_1c_pmw_frequency(ds, product, scan_mode):
 
 def _add_pmw_coordinates(ds, product, scan_mode):
     """Add coordinates to PMW products."""
-    if product.startswith("1C"):
-        ds = _add_1c_pmw_frequency(ds, product, scan_mode)
+    ds = _add_1c_pmw_frequency(ds, product, scan_mode)
+    return ds
+
+
+def _deal_with_pmw_incidence_angle_index(ds):
+    if "incidenceAngleIndex" in ds:
+        if "incidence_angle" in ds.dims and ds.sizes["incidence_angle"] > 1:
+            idx_incidence_angle = ds["incidenceAngleIndex"].isel(along_track=0).astype(int).compute() - 1
+        else:
+            # Some 1C files have bad incidenceAngleIndex values ( )
+            idx_incidence_angle = xr.zeros_like(ds["Tc"].isel(cross_track=0), dtype=int).compute()
+
+        if "incidenceAngle" in ds:
+            ds["incidenceAngle"] = ds["incidenceAngle"].isel(incidence_angle=idx_incidence_angle)
+        if "sunGlintAngle" in ds:
+            ds["sunGlintAngle"] = ds["sunGlintAngle"].isel(incidence_angle=idx_incidence_angle)
+        ds = ds.drop_vars("incidenceAngleIndex")
+
+    #     if "incidenceAngle" in ds:
+    #         ds["incidenceAngle"] = ds["incidenceAngle"].isel(incidence_angle=idx_incidence_angle)
+    #     if "sunGlintAngle" in ds:
+    #         ds["sunGlintAngle"] = ds["sunGlintAngle"].isel(incidence_angle=idx_incidence_angle)
+    # ds = ds.drop_vars("incidenceAngleIndex")
     return ds
 
 
 def set_coordinates(ds, product, scan_mode):
+    # Compute spatial coordinates in memory
+    ds["lon"] = ds["lon"].compute()
+    ds["lat"] = ds["lat"].compute()
+
     # ORBIT objects
     if "cross_track" in list(ds.dims):
         # Ensure valid coordinates
@@ -211,10 +246,12 @@ def set_coordinates(ds, product, scan_mode):
         ds = ds.set_coords("sunLocalTime")
 
     #### PMW
-    # - 1C products
-    if product.startswith("1C"):
+    # - 1B and 1C products
+    if product.startswith("1C") or product.startswith("1B"):
         ds = _add_pmw_coordinates(ds, product, scan_mode)
-
+    # - Deal with incidenceAngleIndex in PMW 1C products
+    if product.startswith("1C"):
+        ds = _deal_with_pmw_incidence_angle_index(ds)
     #### RADAR
     if product in ["2A-DPR", "2A-Ku", "2A-Ka", "2A-PR", "2A-ENV-DPR", "2A-ENV-PR", "2A-ENV-Ka", "2A-ENV-Ku"]:
         ds = _add_radar_coordinates(ds, product, scan_mode)

gpm/dataset/groups_variables.py

@@ -35,6 +35,7 @@
     "Quality",  # 1C-PMW
     "L1CqualityFlag",  # 2A-PMW --> copy of Quality
     "dataQuality",  # 2A-DPR, 1B-GMI/TMI, 2B-CMB
+    "incidenceAngleIndex",  # to deal with TMI S1 scan mode ... dropped variable !
 ]
 
 

gpm/etc/pmw/composites/AMSR2.yaml

@@ -0,0 +1,115 @@
+10 + 18 GHz:
+  R:
+    name: 18.7V
+    vmin: 65
+    vmax: 320
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: true
+  G:
+    name: 18.7H
+    vmin: 65
+    vmax: 320
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: true
+  B:
+    name: 10.65H
+    vmin: 65
+    vmax: 320
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: true
+  global_normalization: false
+18 + 23 GHz:
+  R:
+    name: 23.8V
+    vmin: 65
+    vmax: 320
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: true
+  G:
+    name: 18.7V
+    vmin: 65
+    vmax: 320
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: true
+  B:
+    name: 18.7H
+    vmin: 65
+    vmax: 320
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: true
+  global_normalization: false
+37 + 89 GHz:
+  R:
+    name: 89V
+    vmin: 65
+    vmax: 320
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: true
+  G:
+    name: 89H
+    vmin: 65
+    vmax: 320
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: true
+  B:
+    name: 36.5H
+    vmin: 65
+    vmax: 320
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: true
+  global_normalization: false
+NRL_36.5:
+  R:
+    name: PCT_36.5
+    vmin: 260
+    vmax: 280
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: true
+  G:
+    name: 36.5V
+    vmin: 160
+    vmax: 300
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: false
+  B:
+    name: 36.5H
+    vmin: 180
+    vmax: 300
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: false
+  global_normalization: false
+NRL_89:
+  R:
+    name: PCT_89
+    vmin: 230
+    vmax: 290
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: true
+  G:
+    name: 89H
+    vmin: 255
+    vmax: 290
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: false
+  B:
+    name: 89V
+    vmin: 275
+    vmax: 290
+    vmin_dynamic: false
+    vmax_dynamic: false
+    invert: false
+  global_normalization: false