diff --git a/.gitignore b/.gitignore index 75c6166..e8b71e9 100644 --- a/.gitignore +++ b/.gitignore @@ -115,4 +115,7 @@ _old/ .DS_Store *.zarr *.nc -db_init \ No newline at end of file +db_init +catalog +docker-compose.yaml +nginx.conf \ No newline at end of file diff --git a/README.md b/README.md index 4d775a5..30f7795 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10598417.svg)](https://doi.org/10.5281/zenodo.10598417) - # geolake ## Description @@ -12,13 +10,14 @@ The system has been designed using a cloud-native architecture, based on contain It uses [geokube](https://github.com/CMCC-Foundation/geokube) as an Analytics Engine to perform geospatial operations. -## Authors +## Developer Team + +- [Valentina Scardigno](https://github.com/vale95-eng) +- [Gabriele Tramonte](https://github.com/gtramonte) -**Project Lead**: -[Marco Mancini](https://github.com/km4rcus) +### Former Developers -**Main Developers** +- [Marco Mancini](https://github.com/km4rcus) - [Jakub Walczak](https://github.com/jamesWalczak) - [Mirko Stojiljkovic](https://github.com/MMStojiljkovic) -- [Valentina Scardigno](https://github.com/vale95-eng) diff --git a/catalog/README.md b/catalog/README.md deleted file mode 100644 index 72f90f3..0000000 --- a/catalog/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# geolake-sample-catalog -geolake Catalog Sample diff --git a/catalog/RS_indices.yaml b/catalog/RS_indices.yaml deleted file mode 100644 index e85ecbe..0000000 --- a/catalog/RS_indices.yaml +++ /dev/null @@ -1,40 +0,0 @@ -metadata: - description: >- - Remote Sensing Indices derived from SENTINEL S2A data - contact: - name: Data Deliver System Support Team - email: dds-support@cmcc.it - webpage: https://www.cmcc.it/research-organization/research-divisions/advanced-scientific-computing-division#1553329820238-2055494b-9aa6 - label: Remote Sensing Indices from Sentinel S2A - image: null - doi: null - update_frequency: null - license: null - publication_date: 2023-11-22 - related_data: null - -sources: - 10m: - description: Remote Sensing Indices at 10m - metadata: - role: public - filters: - - name: pasture - user_defined: T - label: Pasture - driver: geokube_netcdf - args: - path: '{{ CATALOG_DIR }}/datasets/RS_indices/*/10m/*.nc' - pattern: '{{ CATALOG_DIR }}/datasets/RS_indices/{pasture}/10m/{}.nc' - field_id: '{standard_name}' - mapping: - NDVI: {'name': 'NDVI', 'description': 'Normalized Difference Vegetation Index'} - NDWI: {'name': 'NDWI', 'description': 'Normalized Difference Water Index'} - GLI: {'name': 'GLI', 'description': 'Green Leaf Index'} - GCI: {'name': 'GCI', 'description': 'Green Chlorophyll Index'} - RGR: {'name': 'RGR', 'description': 'Red-Green Ratio'} - metadata_caching: false - metadata_cache_path: '{{ CACHE_DIR }}/s2-indices-10m.cache' - xarray_kwargs: - parallel: true - decode_coords: 'all' \ No newline at end of file diff --git a/catalog/cache.py b/catalog/cache.py deleted file mode 100644 index 15bca03..0000000 --- a/catalog/cache.py +++ /dev/null @@ -1,22 +0,0 @@ -import argparse -import intake - -parser = argparse.ArgumentParser( - prog="Cache generator", - description="The script generating cache for the catalog", -) -parser.add_argument( - "--cachedir", - type=str, - help="Directory where the cache should be saved. Default: .cache", - default=".cache", -) - -if __name__ == "__main__": - args = parser.parse_args() - catalog = intake.open_catalog("catalog.yaml") - for ds in list(catalog): - for p in list(catalog[ds]): - print(f"dataset: {ds} product: {p}:") - catalog = catalog(CACHE_DIR=args.cachedir) - kube = catalog[ds][p].read() \ No newline at end of file diff --git a/catalog/catalog.yaml b/catalog/catalog.yaml deleted file mode 100644 index 19e5913..0000000 --- a/catalog/catalog.yaml +++ /dev/null @@ -1,23 +0,0 @@ -metadata: - version: 0.1 - parameters: - CACHE_DIR: - type: str - description: folder to store metadata cache files - default: .cache - -sources: - era5-downscaled: - driver: yaml_file_cat - args: - path: '{{ CATALOG_DIR }}/era5_downscaled.yaml' - - thi: - driver: yaml_file_cat - args: - path: '{{ CATALOG_DIR }}/thi.yaml' - - rs-indices: - driver: yaml_file_cat - args: - path: '{{ CATALOG_DIR }}/RS_indices.yaml' diff --git a/catalog/datasets/RS_indices/Donnola/10m/regular.nc b/catalog/datasets/RS_indices/Donnola/10m/regular.nc deleted file mode 100644 index 18e011a..0000000 Binary files a/catalog/datasets/RS_indices/Donnola/10m/regular.nc and /dev/null differ diff --git a/catalog/datasets/THI/20240101.nc b/catalog/datasets/THI/20240101.nc deleted file mode 100644 index f764727..0000000 Binary files a/catalog/datasets/THI/20240101.nc and /dev/null differ diff --git a/catalog/datasets/era5_downscaled.nc b/catalog/datasets/era5_downscaled.nc deleted file mode 100644 index 4d569d9..0000000 Binary files a/catalog/datasets/era5_downscaled.nc and /dev/null differ diff --git a/catalog/era5_downscaled.yaml b/catalog/era5_downscaled.yaml deleted file mode 100644 index ed7fe9f..0000000 --- a/catalog/era5_downscaled.yaml +++ /dev/null @@ -1,12 +0,0 @@ -metadata: - description: >- - This dataset is related to ERA5 downscaled over Italy at 2km. - -sources: - hourly: - description: ERA5 downscaled at 2km over italy hourly. - driver: geokube_netcdf - args: - path: '{{ CATALOG_DIR }}/datasets/era5_downscaled.nc' - metadata_caching: true - metadata_cache_path: '{{ CACHE_DIR }}/era5_downscaled.cache' diff --git a/catalog/thi.yaml b/catalog/thi.yaml deleted file mode 100644 index 6bc9c97..0000000 --- a/catalog/thi.yaml +++ /dev/null @@ -1,37 +0,0 @@ -metadata: - description: >- - Thermohygrometric Indices derived from MISTRAL COSMO-2I data - contact: - name: Data Deliver System Support Team - email: dds-support@cmcc.it - webpage: https://www.cmcc.it/research-organization/research-divisions/advanced-scientific-computing-division#1553329820238-2055494b-9aa6 - label: Thermohygrometric Indices over Italy - image: null - doi: null - update_frequency: null - license: null - publication_date: 2023-06-19 - related_data: null - -sources: - hourly: - description: Hourly Thermohygrometric Indices - metadata: - role: public - filters: - - name: date - user_defined: T - label: Date - driver: geokube_netcdf - args: - path: '{{ CATALOG_DIR }}/datasets/THI/*.nc' - pattern: '{{ CATALOG_DIR }}/datasets/THI/{date}.nc' - field_id: '{standard_name}' - mapping: - THI_ext: {'name': 'external_thermohygrometric_index', 'description': 'External Thermohygrometric Index'} - THI_int: {'name': 'internal_thermohygrometric_index', 'description': 'Internal Thermohygrometric Index'} - metadata_caching: false - metadata_cache_path: '{{ CACHE_DIR }}/thi-hourly.cache' - xarray_kwargs: - parallel: true - decode_coords: 'all' \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml deleted file mode 100644 index 81736dd..0000000 --- a/docker-compose.yaml +++ /dev/null @@ -1,120 +0,0 @@ -version: "3.9" -services: - db: - image: postgres:latest - environment: - - POSTGRES_USER=dds - - POSTGRES_PASSWORD=dds - - POSTGRES_DB=dds - ports: - - "5432:5432" - expose: - - 5432 - healthcheck: - test: [ "CMD-SHELL", "pg_isready", "-d", "db_prod" ] - interval: 30s - timeout: 60s - retries: 5 - start_period: 80s - - restore-db: - image: rg.fr-par.scw.cloud/geodds-production/backup:v0.1a2 - depends_on: - db: - condition: service_healthy - command: - - /bin/bash - - -c - - | - /db_init/restore.sh - tail -f /dev/null - volumes: - - ./db_init:/db_init - environment: - - POSTGRES_HOST=db - - FILENAME=postgres_localhost-2024_10_03_09_53_46-dump-blank.sql - - POSTGRES_USER=dds - - POSTGRES_PASSWORD=dds - - POSTGRES_DB=dds - - catalog: - image: rg.fr-par.scw.cloud/geolake/geolake-datastore:latest - build: ./datastore - volumes: - - ./catalog:/catalog - command: - - /bin/bash - - -c - - | - trap : TERM INT - sleep infinity & - wait - - api: - image: geolago-api:dev - build: ./api - command: - - ./../wait-for-it.sh - - $(BROKER_SERVICE_HOST):5672 - - -- - - uvicorn - - main:app - - --host - - 0.0.0.0 - - --port - - '80' - environment: - - DB_SERVICE_HOST=db - - DB_SERVICE_PORT=5432 - - BROKER_SERVICE_HOST=broker - - POSTGRES_USER=dds - - POSTGRES_PASSWORD=dds - - POSTGRES_DB=dds - - CATALOG_PATH=/catalog/catalog.yaml - - CACHE_PATH=/catalog/.cache - - MESSAGE_SEPARATOR='\' - volumes: - - ./catalog:/catalog - - ./downloads:/downloads - - ./catalog/datasets:/data - expose: - - '8080' - ports: - - '8080:80' - - broker: - image: rabbitmq:3.12-management-alpine - ports: - - "5672:5672" - - "15672:15672" - expose: - - 5672 - - executor: - image: geolago-executor:dev - build: ./executor - command: - - ./../wait-for-it.sh - - $(BROKER_SERVICE_HOST):5672 - - -- - - python - - main.py - environment: - - BROKER_SERVICE_HOST=broker - - DB_SERVICE_HOST=db - - DB_SERVICE_PORT=5432 - - POSTGRES_USER=dds - - POSTGRES_PASSWORD=dds - - POSTGRES_DB=dds - - EXECUTOR_TYPES=query - - MESSAGE_SEPARATOR='\' - - SLEEP_SEC=10 - - RESULT_CHECK_RETRIES=360 - - CATALOG_PATH=/catalog/catalog.yaml - - CACHE_PATH=/catalog/.cache - - HDF5_USE_FILE_LOCKING='FALSE' - - LD_LIBRARY_PATH=/urs/lib/x86_64-linux-gnu - volumes: - - ./catalog:/catalog - - ./downloads:/downloads - - ./catalog/datasets:/data \ No newline at end of file