Merge pull request #32 from GEOSYS/dev

build(release) : v0.0.1

CHANGELOG.md

@@ -5,6 +5,17 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
+## [0.0.1] - 2023-12-15
+
+### Added
+
+- Auth datacube `earthdaily.earthdatastore.Auth().datacube(...)` function now manage multiple collections.
+- Uses `fields` to query only assets asked over Skyfox/EDS (enhance performance).
+
+### Changed
+
+- `asset_mapper` has now all assets from available collections.
+
 ## [0.0.1-rc9] - 2023-12-12
 
 ### Added

earthdaily/__init__.py

@@ -1,3 +1,3 @@
 from . import earthdatastore, datasets
 
-__version__ = "0.0.1-rc9"
+__version__ = "0.0.1"

earthdaily/earthdatastore/__init__.py

@@ -13,7 +13,7 @@
 from pystac_client import Client
 
 from . import _scales_collections, cube_utils, mask
-from .cube_utils import datacube, metacube
+from .cube_utils import datacube, metacube, _datacubes
 
 __all__ = ["datacube", "metacube", "xr"]
 
@@ -398,6 +398,7 @@ def _update_search_kwargs_for_ag_cloud_mask(self, search_kwargs, collections):
         search_kwargs[target_param] = query
         return search_kwargs
 
+    @_datacubes
     def datacube(
         self,
         collections: str | list,
@@ -415,27 +416,103 @@ def datacube(
         cross_calibration_collection: (None | str) = None,
         **kwargs,
     ) -> xr.Dataset:
+        """
+        Create a datacube.
+
+        Parameters
+        ----------
+        collections : str | list
+            If several collections, the first collection will be the reference collection (for spatial resolution).
+        datetime: Either a single datetime or datetime range used to filter results.
+            You may express a single datetime using a :class:`datetime.datetime`
+            instance, a `RFC 3339-compliant <https://tools.ietf.org/html/rfc3339>`__
+            timestamp, or a simple date string (see below). Instances of
+            :class:`datetime.datetime` may be either
+            timezone aware or unaware. Timezone aware instances will be converted to
+            a UTC timestamp before being passed
+            to the endpoint. Timezone unaware instances are assumed to represent UTC
+            timestamps. You may represent a
+            datetime range using a ``"/"`` separated string as described in the
+            spec, or a list, tuple, or iterator
+            of 2 timestamps or datetime instances. For open-ended ranges, use either
+            ``".."`` (``'2020-01-01:00:00:00Z/..'``,
+            ``['2020-01-01:00:00:00Z', '..']``) or a value of ``None``
+            (``['2020-01-01:00:00:00Z', None]``).
+
+            If using a simple date string, the datetime can be specified in
+            ``YYYY-mm-dd`` format, optionally truncating
+            to ``YYYY-mm`` or just ``YYYY``. Simple date strings will be expanded to
+            include the entire time period, for example:
+
+            - ``2017`` expands to ``2017-01-01T00:00:00Z/2017-12-31T23:59:59Z``
+            - ``2017-06`` expands to ``2017-06-01T00:00:00Z/2017-06-30T23:59:59Z``
+            - ``2017-06-10`` expands to
+              ``2017-06-10T00:00:00Z/2017-06-10T23:59:59Z``
+
+            If used in a range, the end of the range expands to the end of that
+            day/month/year, for example:
+
+            - ``2017/2018`` expands to
+              ``2017-01-01T00:00:00Z/2018-12-31T23:59:59Z``
+            - ``2017-06/2017-07`` expands to
+              ``2017-06-01T00:00:00Z/2017-07-31T23:59:59Z``
+            - ``2017-06-10/2017-06-11`` expands to
+              ``2017-06-10T00:00:00Z/2017-06-11T23:59:59Z``
+        assets : None | list | dict, optional
+            DESCRIPTION. The default is None.
+        intersects : (gpd.GeoDataFrame, str(wkt), dict(json)), optional
+            DESCRIPTION. The default is None.
+        bbox : TYPE, optional
+            DESCRIPTION. The default is None.
+        mask_with : (None, str), optional
+            "native" mask, or "ag_cloud_mask".
+            The default is None.
+        mask_statistics : bool | int, optional
+            DESCRIPTION. The default is False.
+        clear_cover : (int, float), optional
+            Percent of clear data above a field (from 0 to 100).
+            The default is None.
+        prefer_alternate : (str, False), optional
+            Uses the alternate/download href instead of the default href.
+            The default is "download".
+        search_kwargs : dict, optional
+            DESCRIPTION. The default is {}.
+        add_default_scale_factor : bool, optional
+            DESCRIPTION. The default is True.
+        common_band_names : TYPE, optional
+            DESCRIPTION. The default is True.
+        cross_calibration_collection : (None | str), optional
+            DESCRIPTION. The default is None.
+        **kwargs : TYPE
+            DESCRIPTION.
+         : TYPE
+            DESCRIPTION.
+
+        Raises
+        ------
+        ValueError
+            DESCRIPTION.
+        Warning
+            DESCRIPTION.
+
+        Returns
+        -------
+        xr_datacube : TYPE
+            DESCRIPTION.
+
+        """
         if isinstance(collections, str):
             collections = [collections]
 
-        if mask_with and common_band_names:
-            if isinstance(collections, list):
-                if len(collections) > 1:
-                    raise ValueError(
-                        "Mask_with and assets_mapping only manage one collection at a time."
-                    )
         if mask_with:
             if mask_with not in mask._available_masks:
                 raise ValueError(
                     f"Specified mask '{mask_with}' is not available. Currently available masks provider are : {mask._available_masks}"
                 )
-            collection = (
-                collections[0] if isinstance(collections, list) else collections
-            )
 
             if mask_with == "ag_cloud_mask":
                 search_kwargs = self._update_search_kwargs_for_ag_cloud_mask(
-                    search_kwargs, collections
+                    search_kwargs, collections[0]
                 )
         if intersects is not None:
             intersects = cube_utils.GeometryManager(intersects).to_geopandas()
@@ -450,7 +527,10 @@ def datacube(
         )
 
         xcal_items = None
-        if isinstance(cross_calibration_collection, str):
+        if (
+            isinstance(cross_calibration_collection, str)
+            and cross_calibration_collection != collections[0]
+        ):
             try:
                 xcal_items = self.search(
                     collections="eda-cross-calibration",
@@ -484,10 +564,10 @@ def datacube(
                     category=Warning,
                 )
             if mask_with == "native":
-                mask_with = mask._native_mask_def_mapping.get(collection, None)
+                mask_with = mask._native_mask_def_mapping.get(collections[0], None)
                 if mask_with is None:
                     raise ValueError(
-                        f"Sorry, there's no native mask available for {collection}. Only these collections have native cloudmask : {list(mask._native_mask_mapping.keys())}."
+                        f"Sorry, there's no native mask available for {collections[0]}. Only these collections have native cloudmask : {list(mask._native_mask_mapping.keys())}."
                     )
             mask_kwargs = dict(mask_statistics=mask_statistics)
 
@@ -507,7 +587,7 @@ def datacube(
                 acm_datacube = cube_utils._match_xy_dims(acm_datacube, xr_datacube)
                 mask_kwargs.update(acm_datacube=acm_datacube)
             else:
-                mask_assets = mask._native_mask_asset_mapping[collection]
+                mask_assets = mask._native_mask_asset_mapping[collections[0]]
                 if "groupby_date" in kwargs:
                     kwargs["groupby_date"] = "max"
                 if "resolution" in kwargs:

earthdaily/earthdatastore/cube_utils/__init__.py

@@ -13,10 +13,34 @@
 from .harmonizer import Harmonizer
 from .asset_mapper import AssetMapper
 import rioxarray
+from functools import wraps
 
 __all__ = ["GeometryManager", "rioxarray", "zonal_stats", "zonal_stats_numpy"]
 
 
+def _datacubes(method):
+    @wraps(method)
+    def _impl(self, *args, **kwargs):
+        collections = kwargs.get("collections", args[0])
+        if isinstance(collections, list) and len(collections) > 1:
+            if "collections" in kwargs.keys():
+                kwargs.pop("collections")
+            else:
+                args = args[1:]
+            datacubes = []
+            for idx, collection in enumerate(collections):
+                datacube = method(self, collections=collection, *args, **kwargs)
+                if idx == 0:
+                    kwargs["geobox"] = datacube.odc.geobox
+                datacubes.append(datacube)
+            datacube = metacube(*datacubes)
+        else:
+            datacube = method(self, *args, **kwargs)
+        return datacube
+
+    return _impl
+
+
 def _match_xy_dims(src, dst, resampling=Resampling.nearest):
     if src.dims != dst.dims:
         src = src.rio.reproject_match(dst, resampling=resampling)
@@ -384,7 +408,6 @@ def metacube(*cubes, concat_dim="time", by="time.date", how="mean"):
                 cubes[idx][data_var] = cubes[idx][data_var].where(
                     cubes[idx][data_var] == np.nan, other=np.nan
                 )
-
-    cube = xr.concat([_drop_unfrozen_coords(cube) for cube in cubes], dim=concat_dim)
+    cube = xr.concat([cube for cube in cubes], dim=concat_dim)
     cube = _groupby(cube, by=by, how=how)
     return _propagade_rio(cubes[0], cube)

earthdaily/earthdatastore/cube_utils/asset_mapper/__init__.py

@@ -1,15 +1,18 @@
-from . import _asset_mapper_config
+import os
+import json
+
+__pathFile = os.path.dirname(os.path.realpath(__file__))
+__asset_mapper_config_path = os.path.join(__pathFile, "asset_mapper_config.json")
+_asset_mapper_config = json.load(open(__asset_mapper_config_path))
 
 
 class AssetMapper:
     def __init__(self):
-        self.available_collections = list(
-            _asset_mapper_config.asset_mapper_collections.keys()
-        )
+        self.available_collections = list(_asset_mapper_config.keys())
 
     def collection_mapping(self, collection):
         if self._collection_exists(collection, raise_warning=True):
-            return _asset_mapper_config.asset_mapper_collections[collection]
+            return _asset_mapper_config[collection]
 
     def _collection_exists(self, collection, raise_warning=False):
         exists = True if collection in self.available_collections else False

earthdaily/earthdatastore/cube_utils/asset_mapper/__update_asset_mapper_config_json.py

@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Dec 15 09:03:08 2023
+
+@author: nkk
+"""
+
+import earthdaily
+import json
+
+eds = earthdaily.earthdatastore.Auth()
+asset_mapper_path = (
+    earthdaily.earthdatastore.cube_utils.asset_mapper.__asset_mapper_config_path
+)
+asset_mapper_config = (
+    earthdaily.earthdatastore.cube_utils.asset_mapper.__asset_mapper_config
+)
+
+for collection in eds.explore():
+    try:
+        assets_name = list(eds.explore(collection).item.assets.keys())
+    except AttributeError:
+        print(f"collection {collection} has no items")
+        continue
+    for asset_name in assets_name:
+        if asset_mapper_config.get(collection) is None:
+            asset_mapper_config[collection] = [{}]
+        if asset_name not in asset_mapper_config[collection][0].values():
+            asset_mapper_config[collection][0][asset_name] = asset_name
+
+with open(asset_mapper_path, "w", encoding="utf-8") as f:
+    json.dump(asset_mapper_config, f, ensure_ascii=False, indent=4)