From 1b6cea05f5e9d21f960555a9e138ad7912f7113b Mon Sep 17 00:00:00 2001 From: Gabriele Tramonte Date: Fri, 4 Oct 2024 11:33:32 +0200 Subject: [PATCH] Updating geolake images with the latest geokube release to enable support for zarr format --- .gitignore | 5 +- api/Dockerfile | 2 +- datastore/Dockerfile | 2 +- docker-compose.yaml | 183 ++++++++++++++++++++++++++----------------- drivers/Dockerfile | 3 +- executor/Dockerfile | 2 +- 6 files changed, 118 insertions(+), 79 deletions(-) diff --git a/.gitignore b/.gitignore index 1856070..75c6166 100644 --- a/.gitignore +++ b/.gitignore @@ -112,4 +112,7 @@ venv.bak/ _catalogs/ _old/ -.DS_Store \ No newline at end of file +.DS_Store +*.zarr +*.nc +db_init \ No newline at end of file diff --git a/api/Dockerfile b/api/Dockerfile index ad73842..e5016fa 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -6,7 +6,7 @@ RUN apt update && apt install -y cron curl WORKDIR /app COPY requirements.txt /code/requirements.txt -RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt +RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt --break-system-packages COPY app /app EXPOSE 80 diff --git a/datastore/Dockerfile b/datastore/Dockerfile index 7e051cc..77fc265 100644 --- a/datastore/Dockerfile +++ b/datastore/Dockerfile @@ -3,7 +3,7 @@ ARG TAG=latest FROM $REGISTRY/geolake-drivers:$TAG COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir -r /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt --break-system-packages COPY ./datastore /app/datastore COPY ./workflow /app/workflow COPY ./dbmanager /app/dbmanager diff --git a/docker-compose.yaml b/docker-compose.yaml index 995b7ca..81736dd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,85 +1,120 @@ -version: "3" +version: "3.9" services: - api: - build: - context: ./ - dockerfile: ./api/Dockerfile + db: + image: postgres:latest + environment: + - POSTGRES_USER=dds + - POSTGRES_PASSWORD=dds + - POSTGRES_DB=dds ports: - - "8080:80" + - "5432:5432" + expose: + - 5432 + healthcheck: + test: [ "CMD-SHELL", "pg_isready", "-d", "db_prod" ] + interval: 30s + timeout: 60s + retries: 5 + start_period: 80s + + restore-db: + image: rg.fr-par.scw.cloud/geodds-production/backup:v0.1a2 depends_on: - - broker - - db - links: - - broker - - db - environment: - CATALOG_PATH: /code/app/resources/catalogs/catalog.yaml - POSTGRES_DB: dds - POSTGRES_USER: dds - POSTGRES_PASSWORD: dds - POSTGRES_HOST: db - POSTGRES_PORT: 5432 + db: + condition: service_healthy + command: + - /bin/bash + - -c + - | + /db_init/restore.sh + tail -f /dev/null volumes: - - downloads:/downloads:ro - command: ["./wait-for-it.sh", "broker:5672", "--", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "80"] - executor: - build: - context: ./ - dockerfile: ./executor/Dockerfile - depends_on: - - broker - - db - links: - - broker - - db - ports: - - 8787:8787 + - ./db_init:/db_init environment: - EXECUTOR_TYPES: query,info,estimate - CATALOG_PATH: /code/app/resources/catalogs/catalog.yaml - POSTGRES_DB: dds - POSTGRES_USER: dds - POSTGRES_PASSWORD: dds - POSTGRES_HOST: db - POSTGRES_PORT: 5432 + - POSTGRES_HOST=db + - FILENAME=postgres_localhost-2024_10_03_09_53_46-dump-blank.sql + - POSTGRES_USER=dds + - POSTGRES_PASSWORD=dds + - POSTGRES_DB=dds + + catalog: + image: rg.fr-par.scw.cloud/geolake/geolake-datastore:latest + build: ./datastore volumes: - - downloads:/downloads:rw - command: ["./wait-for-it.sh", "broker:5672", "--", "python", "./app/main.py"] - broker: - image: rabbitmq:3 - db: - image: postgres:14.1 - restart: always + - ./catalog:/catalog + command: + - /bin/bash + - -c + - | + trap : TERM INT + sleep infinity & + wait + + api: + image: geolago-api:dev + build: ./api + command: + - ./../wait-for-it.sh + - $(BROKER_SERVICE_HOST):5672 + - -- + - uvicorn + - main:app + - --host + - 0.0.0.0 + - --port + - '80' + environment: + - DB_SERVICE_HOST=db + - DB_SERVICE_PORT=5432 + - BROKER_SERVICE_HOST=broker + - POSTGRES_USER=dds + - POSTGRES_PASSWORD=dds + - POSTGRES_DB=dds + - CATALOG_PATH=/catalog/catalog.yaml + - CACHE_PATH=/catalog/.cache + - MESSAGE_SEPARATOR='\' volumes: - - ./db/scripts:/docker-entrypoint-initdb.d + - ./catalog:/catalog + - ./downloads:/downloads + - ./catalog/datasets:/data + expose: + - '8080' ports: - - 5432:5432 - environment: - POSTGRES_DB: dds - POSTGRES_USER: dds - POSTGRES_PASSWORD: dds - POSTGRES_HOST: db - POSTGRES_PORT: 5432 - web: - build: - context: ./ - dockerfile: ./web/Dockerfile + - '8080:80' + + broker: + image: rabbitmq:3.12-management-alpine ports: - - "8080:80" - depends_on: - - db - links: - - db + - "5672:5672" + - "15672:15672" + expose: + - 5672 + + executor: + image: geolago-executor:dev + build: ./executor + command: + - ./../wait-for-it.sh + - $(BROKER_SERVICE_HOST):5672 + - -- + - python + - main.py environment: - CATALOG_PATH: /code/app/resources/catalogs/catalog.yaml - POSTGRES_DB: dds - POSTGRES_USER: dds - POSTGRES_PASSWORD: dds - POSTGRES_HOST: db - POSTGRES_PORT: 5432 + - BROKER_SERVICE_HOST=broker + - DB_SERVICE_HOST=db + - DB_SERVICE_PORT=5432 + - POSTGRES_USER=dds + - POSTGRES_PASSWORD=dds + - POSTGRES_DB=dds + - EXECUTOR_TYPES=query + - MESSAGE_SEPARATOR='\' + - SLEEP_SEC=10 + - RESULT_CHECK_RETRIES=360 + - CATALOG_PATH=/catalog/catalog.yaml + - CACHE_PATH=/catalog/.cache + - HDF5_USE_FILE_LOCKING='FALSE' + - LD_LIBRARY_PATH=/urs/lib/x86_64-linux-gnu volumes: - - downloads:/downloads:ro - command: ["./wait-for-it.sh", "broker:5672", "--", "uvicorn", "web.main:app", "--host", "0.0.0.0", "--port", "80"] - -volumes: - downloads: \ No newline at end of file + - ./catalog:/catalog + - ./downloads:/downloads + - ./catalog/datasets:/data \ No newline at end of file diff --git a/drivers/Dockerfile b/drivers/Dockerfile index 978eaa1..3923caf 100644 --- a/drivers/Dockerfile +++ b/drivers/Dockerfile @@ -1,6 +1,7 @@ ARG REGISTRY=rg.fr-par.scw.cloud/geokube #ARG TAG=v0.2.6b2 -ARG TAG=2024.05.03.10.36 +#ARG TAG=2024.05.03.10.36 +ARG TAG=v0.2.7 FROM $REGISTRY/geokube:$TAG COPY dist/intake_geokube-0.1a0-py3-none-any.whl / diff --git a/executor/Dockerfile b/executor/Dockerfile index e3404e5..3399feb 100644 --- a/executor/Dockerfile +++ b/executor/Dockerfile @@ -7,7 +7,7 @@ RUN apt update && apt install -y cron curl WORKDIR /app COPY requirements.txt /code/requirements.txt -RUN pip install --no-cache-dir -r /code/requirements.txt +RUN pip install --no-cache-dir -r /code/requirements.txt --break-system-packages COPY app /app COPY ./healtcheck.* /opt/