fix: use base64 encoded JSON string of sort keys as pagination token (#323)

**Related Issue(s):**

- #322 

**Description:**
Use base64 encoded JSON string of sort keys as pagination token instead
of comma separated string.

Author: StijnCaerts

CHANGELOG.md

@@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Use base64 encoded JSON string of sort keys as pagination token instead of comma-separated string [#323](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/323)
 
 ## [v3.2.1] - 2024-11-14
 

stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py

@@ -1,6 +1,7 @@
 """Database logic."""
 
 import asyncio
+import json
 import logging
 import os
 from base64 import urlsafe_b64decode, urlsafe_b64encode
@@ -660,7 +661,7 @@ async def execute_search(
         search_after = None
 
         if token:
-            search_after = urlsafe_b64decode(token.encode()).decode().split(",")
+            search_after = json.loads(urlsafe_b64decode(token).decode())
 
         query = search.query.to_dict() if search.query else None
 
@@ -700,9 +701,7 @@ async def execute_search(
         next_token = None
         if len(hits) > limit and limit < max_result_window:
             if hits and (sort_array := hits[limit - 1].get("sort")):
-                next_token = urlsafe_b64encode(
-                    ",".join([str(x) for x in sort_array]).encode()
-                ).decode()
+                next_token = urlsafe_b64encode(json.dumps(sort_array).encode()).decode()
 
         matched = (
             es_response["hits"]["total"]["value"]

stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py

@@ -1,6 +1,7 @@
 """Database logic."""
 
 import asyncio
+import json
 import logging
 import os
 from base64 import urlsafe_b64decode, urlsafe_b64encode
@@ -692,7 +693,7 @@ async def execute_search(
         search_after = None
 
         if token:
-            search_after = urlsafe_b64decode(token.encode()).decode().split(",")
+            search_after = json.loads(urlsafe_b64decode(token).decode())
         if search_after:
             search_body["search_after"] = search_after
 
@@ -732,9 +733,7 @@ async def execute_search(
         next_token = None
         if len(hits) > limit and limit < max_result_window:
             if hits and (sort_array := hits[limit - 1].get("sort")):
-                next_token = urlsafe_b64encode(
-                    ",".join([str(x) for x in sort_array]).encode()
-                ).decode()
+                next_token = urlsafe_b64encode(json.dumps(sort_array).encode()).decode()
 
         matched = (
             es_response["hits"]["total"]["value"]