Search optimization and indexing based on datetime

.github/workflows/cicd.yml

@@ -28,6 +28,7 @@ jobs:
           xpack.security.enabled: false
           xpack.security.transport.ssl.enabled: false
           ES_JAVA_OPTS: -Xms512m -Xmx1g
+          action.destructive_requires_name: false
         ports:
           - 9200:9200
 
@@ -44,6 +45,7 @@ jobs:
           xpack.security.enabled: false
           xpack.security.transport.ssl.enabled: false
           ES_JAVA_OPTS: -Xms512m -Xmx1g
+          action.destructive_requires_name: false
         ports:
           - 9400:9400
 
@@ -60,6 +62,7 @@ jobs:
           plugins.security.disabled: true
           plugins.security.ssl.http.enabled: true
           OPENSEARCH_JAVA_OPTS: -Xms512m -Xmx512m
+          action.destructive_requires_name: false
         ports:
           - 9202:9202
 
@@ -120,5 +123,6 @@ jobs:
           ES_PORT: ${{ matrix.backend == 'elasticsearch7' && '9400' || matrix.backend == 'elasticsearch8' && '9200' || '9202' }}
           ES_HOST: 172.17.0.1
           ES_USE_SSL: false
+          DATABASE_REFRESH: true
           ES_VERIFY_CERTS: false
           BACKEND: ${{ matrix.backend == 'elasticsearch7' && 'elasticsearch' || matrix.backend == 'elasticsearch8' && 'elasticsearch' || 'opensearch' }}

CHANGELOG.md

@@ -8,6 +8,32 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Added 
+
+- Added comprehensive index management system with dynamic selection and insertion strategies for improved performance and scalability [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405)
+- Added `ENABLE_DATETIME_INDEX_FILTERING` environment variable to enable datetime-based index selection using collection IDs. Requires indexes in format: `STAC_ITEMS_INDEX_PREFIX_collection-id_start_year-start_month-start_day-end_year-end_month-end_day`, e.g. `items_sentinel-2-l2a_2025-06-06-2025-09-22`. Default is `false`. [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405)
+- Added `DATETIME_INDEX_MAX_SIZE_GB` environment variable to set maximum size limit in GB for datetime-based indexes. When an index exceeds this size, a new time-partitioned index will be created. Default is `25` GB. Only applies when `ENABLE_DATETIME_INDEX_FILTERING` is enabled. [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405)
+- Added search engine adapter system with support for both Elasticsearch and OpenSearch [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405):
+  - `SearchEngineAdapter` base class with engine-specific implementations
+  - `ElasticsearchAdapter` and `OpenSearchAdapter` with tailored index creation methods
+  - Automatic engine type detection based on client class
+  - `SearchEngineAdapterFactory` for creating appropriate adapters
+- Added datetime-based index selection strategies with caching support [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405):
+  - `AsyncDatetimeBasedIndexSelector` and `SyncDatetimeBasedIndexSelector` for temporal filtering
+  - `IndexCacheManager` with configurable TTL-based cache expiration (default 1 hour)
+  - `AsyncIndexAliasLoader` and `SyncIndexAliasLoader` for alias management
+  - `UnfilteredIndexSelector` as fallback for returning all available indexes
+- Added index insertion strategies with automatic partitioning [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405):
+  - Simple insertion strategy (`AsyncSimpleIndexInserter`, `SyncSimpleIndexInserter`) for traditional single-index-per-collection approach
+  - Datetime-based insertion strategy (`AsyncDatetimeIndexInserter`, `SyncDatetimeIndexInserter`) with time-based partitioning
+  - Automatic index size monitoring and splitting when limits exceeded
+  - Handling of chronologically early data and bulk operations
+- Added index management utilities [#405](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/405):
+  - `IndexSizeManager` for size monitoring and overflow handling
+  - `DatetimeIndexManager` for datetime-based index operations
+  - Factory patterns (`IndexInsertionFactory`, `IndexSelectorFactory`) for strategy creation based on configuration
+
+
 ## [v6.1.0] - 2025-07-24
 
 ### Added

Makefile

@@ -27,7 +27,7 @@ run_os = docker compose \
 .PHONY: image-deploy-es
 image-deploy-es:
 	docker build -f dockerfiles/Dockerfile.dev.es -t stac-fastapi-elasticsearch:latest .
-	
+
 .PHONY: image-deploy-os
 image-deploy-os:
 	docker build -f dockerfiles/Dockerfile.dev.os -t stac-fastapi-opensearch:latest .
@@ -71,14 +71,19 @@ test-opensearch:
 	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest'
 	docker compose down
 
-.PHONY: test
-test:
-	-$(run_es) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest --cov=stac_fastapi --cov-report=term-missing'
+.PHONY: test-datetime-filtering-es
+test-datetime-filtering-es:
+	-$(run_es) /bin/bash -c 'export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh elasticsearch:9200 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering'
 	docker compose down
 
-	-$(run_os) /bin/bash -c 'export && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest --cov=stac_fastapi --cov-report=term-missing'
+.PHONY: test-datetime-filtering-os
+test-datetime-filtering-os:
+	-$(run_os) /bin/bash -c 'export ENABLE_DATETIME_INDEX_FILTERING=true && ./scripts/wait-for-it-es.sh opensearch:9202 && cd stac_fastapi/tests/ && pytest -s --cov=stac_fastapi --cov-report=term-missing -m datetime_filtering'
 	docker compose down
 
+.PHONY: test
+test: test-elasticsearch test-datetime-filtering-es test-opensearch test-datetime-filtering-os
+
 .PHONY: run-database-es
 run-database-es:
 	docker compose run --rm elasticsearch