feat: pagination using token

Author: sbrunato

pyproject.toml

@@ -8,7 +8,7 @@ license = { file = "LICENSE" }
 requires-python = ">= 3.9"
 dependencies = [
     "attr",
-    "eodag[all-providers] == 4.0.0a1",
+    "eodag[all-providers] @ git+https://github.com/CS-SI/eodag.git@develop",
     "fastapi",
     "geojson",
     "geojson-pydantic",
@@ -65,7 +65,7 @@ explicit_package_bases = true
 exclude = ["tests", ".venv"]
 
 [[tool.mypy.overrides]]
-module = ["pygeofilter", "pygeofilter.*", "stac_fastapi", "stac_fastapi.*"]
+module = ["geojson", "pygeofilter", "pygeofilter.*", "stac_fastapi", "stac_fastapi.*"]
 ignore_missing_imports = true
 
 [tool.pytest.ini_options]

stac_fastapi/eodag/core.py

@@ -41,12 +41,13 @@
 from stac_pydantic.links import Relations
 from stac_pydantic.shared import MimeTypes
 
-from eodag import SearchResult
+from eodag import EOProduct, SearchResult
 from eodag.plugins.search.build_search_result import ECMWFSearch
 from eodag.utils import deepcopy, get_geometry_from_various
 from eodag.utils.exceptions import NoMatchingCollection as EodagNoMatchingCollection
 from stac_fastapi.eodag.client import CustomCoreClient
 from stac_fastapi.eodag.config import get_settings
+from stac_fastapi.eodag.constants import DEFAULT_ITEMS_PER_PAGE
 from stac_fastapi.eodag.cql_evaluate import EodagEvaluator
 from stac_fastapi.eodag.errors import NoMatchingCollection, ResponseSearchError
 from stac_fastapi.eodag.models.links import (
@@ -182,15 +183,18 @@ def _search_base(self, search_request: BaseSearchPostRequest, request: Request)
         else:
             raise HTTPException(status_code=400, detail="A collection is required")
 
-        # get products by ids
         if ids := eodag_args.pop("ids", []):
+            # get products by ids
             search_result = SearchResult([])
             for item_id in ids:
                 eodag_args["id"] = item_id
                 search_result.extend(request.app.state.dag.search(validate=validate, **eodag_args))
             search_result.number_matched = len(search_result)
+        elif eodag_args.get("token") and eodag_args.get("provider"):
+            # search with pagination
+            search_result = eodag_search_next_page(request.app.state.dag, eodag_args)
         else:
-            # search without ids
+            # search without ids or pagination
             search_result = request.app.state.dag.search(validate=validate, **eodag_args)
 
         if search_result.errors and not len(search_result):
@@ -207,23 +211,22 @@ def _search_base(self, search_request: BaseSearchPostRequest, request: Request)
             )
             features.append(feature)
 
-        collection = ItemCollection(
+        feature_collection = ItemCollection(
             type="FeatureCollection",
             features=features,
             numberMatched=search_result.number_matched,
             numberReturned=len(features),
         )
 
         # pagination
-        next_page = None
-        if hasattr(search_result, "next_page_token_key"):
-            next_page = search_result.next_page_token_key.split(":", 1)[1]
-
-        collection["links"] = PagingLinks(
+        if "provider" not in request.state.eodag_args and len(search_result) > 0:
+            request.state.eodag_args["provider"] = search_result[-1].provider
+        feature_collection["links"] = PagingLinks(
             request=request,
-            next=next_page,
+            next=search_result.next_page_token,
+            federation_backend=request.state.eodag_args.get("provider"),
         ).get_links(request_json=request_json, extensions=extension_names)
-        return collection
+        return feature_collection
 
     async def all_collections(
         self,
@@ -730,3 +733,36 @@ def add_error(error_message: str) -> None:
         raise ValidationError.from_exception_data(title="stac-fastapi-eodag", line_errors=errors)
 
     return cql_args
+
+
+def eodag_search_next_page(dag, eodag_args):
+    """Perform an eodag search with pagination.
+
+    :param dag: The EODAG instance.
+    :param eodag_args: The EODAG search arguments.
+    :returns: The search result for the next page.
+    """
+    eodag_args = eodag_args.copy()
+    next_page_token = eodag_args.pop("token", None)
+    provider = eodag_args.get("provider")
+    if not next_page_token or not provider:
+        raise HTTPException(
+            status_code=500, detail="Missing required token and federation backend for next page search."
+        )
+    search_plugin = next(dag._plugins_manager.get_search_plugins(provider=provider))
+    next_page_token_key = getattr(search_plugin.config, "pagination", {}).get("next_page_token_key", "page")
+    eodag_args.pop("count", None)
+    search_result = SearchResult(
+        [EOProduct(provider, {"id": "_"})] * int(eodag_args.get("items_per_page", DEFAULT_ITEMS_PER_PAGE)),
+        next_page_token=next_page_token,
+        next_page_token_key=next_page_token_key,
+        search_params=eodag_args,
+        raise_errors=eodag_args.pop("raise_errors", None),
+    )
+    search_result._dag = dag
+    try:
+        search_result = next(search_result.next_page())
+    except StopIteration:
+        logger.info("StopIteration encountered during next page search.")
+        search_result = SearchResult([])
+    return search_result

stac_fastapi/eodag/models/links.py

@@ -21,11 +21,14 @@
 from urllib.parse import ParseResult, parse_qs, unquote, urlencode, urljoin, urlparse
 
 import attr
+import geojson
 from stac_fastapi.types.requests import get_base_url
 from stac_pydantic.links import Relations
 from stac_pydantic.shared import MimeTypes
 from starlette.requests import Request
 
+from eodag.utils import update_nested_dict
+
 # These can be inferred from the item/collection so they aren't included in the database
 # Instead they are dynamically generated when querying the database using the classes defined below
 INFERRED_LINK_RELS = ["self", "item", "collection"]
@@ -139,14 +142,25 @@ def get_links(
 class PagingLinks(BaseLinks):
     """Create links for paging."""
 
-    next: Optional[int] = attr.ib(kw_only=True, default=None)
+    next: Optional[str] = attr.ib(kw_only=True, default=None)
+    federation_backend: Optional[str] = attr.ib(kw_only=True, default=None)
 
     def link_next(self) -> Optional[dict[str, Any]]:
         """Create link for next page."""
         if self.next is not None:
             method = self.request.method
+            federation_backend_dict = (
+                {"query": {"federation:backends": {"eq": self.federation_backend}}} if self.federation_backend else {}
+            )
             if method == "GET":
-                href = merge_params(self.url, {"token": [str(self.next)]})
+                params_update_dict: dict[str, list[str]] = {"token": [str(self.next)]}
+                if "query" in self.request.query_params:
+                    params_update_dict["query"] = [
+                        geojson.dumps(
+                            geojson.loads(self.request.query_params["query"]) | federation_backend_dict["query"]
+                        )
+                    ]
+                href = merge_params(self.url, params_update_dict)
                 return {
                     "rel": Relations.next.value,
                     "type": MimeTypes.geojson.value,
@@ -155,12 +169,15 @@ def link_next(self) -> Optional[dict[str, Any]]:
                     "title": "Next page",
                 }
             if method == "POST":
+                post_body = update_nested_dict(
+                    self.request.state.postbody, {"token": self.next} | federation_backend_dict
+                )
                 return {
                     "rel": Relations.next,
                     "type": MimeTypes.geojson,
                     "method": method,
                     "href": f"{self.request.url}",
-                    "body": {**self.request.state.postbody, "page": self.next},
+                    "body": post_body,
                     "title": "Next page",
                 }