From 7a060c6570d72020ac74b20437f20f5b871eed5b Mon Sep 17 00:00:00 2001 From: Valentina Scardigno Date: Wed, 8 May 2024 08:04:23 +0000 Subject: [PATCH 1/3] Fix query parameter in estimate for datastore --- datastore/datastore/datastore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datastore/datastore/datastore.py b/datastore/datastore/datastore.py index 20a5f27..2525c02 100644 --- a/datastore/datastore/datastore.py +++ b/datastore/datastore/datastore.py @@ -50,7 +50,7 @@ def __init__(self) -> None: @log_execution_time(_LOG) def get_cached_product_or_read( - self, dataset_id: str, product_id: str + self, dataset_id: str, product_id: str, query: GeoQuery | None = None ) -> DataCube | Dataset: """Get product from the cache instead of loading files indicated in the catalog if `metadata_caching` set to `True`. @@ -81,7 +81,7 @@ def get_cached_product_or_read( ) return self.catalog(CACHE_DIR=self.cache_dir)[dataset_id][ product_id - ].read_chunked() + ].get(geoquery=query, compute=False).read_chunked() return self.cache[dataset_id][product_id] @log_execution_time(_LOG) @@ -389,7 +389,7 @@ def estimate( # NOTE: we always use catalog directly and single product cache self._LOG.debug("loading product...") # NOTE: for estimation we use cached products - kube = self.get_cached_product_or_read(dataset_id, product_id) + kube = self.get_cached_product_or_read(dataset_id, product_id, query=query) self._LOG.debug("original kube len: %s", len(kube)) return Datastore._process_query(kube, geoquery, False).nbytes From 95d7ad569c9040b705cddc94c45da8c82cdf7d4e Mon Sep 17 00:00:00 2001 From: Gabriele Tramonte Date: Wed, 8 May 2024 10:09:44 +0200 Subject: [PATCH 2/3] update to docker files and workflow --- .github/workflows/build_on_pull_request.yaml | 30 +++++++++++++------- datastore/Dockerfile | 6 ++-- datastore/requirements.txt | 3 +- drivers/Dockerfile | 6 ++-- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_on_pull_request.yaml b/.github/workflows/build_on_pull_request.yaml index f32be5a..92f1dca 100644 --- a/.github/workflows/build_on_pull_request.yaml +++ b/.github/workflows/build_on_pull_request.yaml @@ -20,9 +20,11 @@ jobs: build --user - name: Build a binary wheel and a source for drivers - run: python3 -m build ./drivers + run: python3 -m build ./drivers - name: Set Docker image tag name run: echo "TAG=$(date +'%Y.%m.%d.%H.%M')" >> $GITHUB_ENV + - name: TAG ECHO + run: echo ${{ env.TAG }} - name: Login to Scaleway Container Registry uses: docker/login-action@v2 with: @@ -30,48 +32,56 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} registry: ${{ vars.DOCKER_REGISTRY }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v2 - name: Build and push drivers - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ./drivers file: ./drivers/Dockerfile push: true + cache-from: type=gha + cache-to: type=gha,mode=max build-args: | REGISTRY=${{ vars.GEOKUBE_REGISTRY }} tags: | ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:${{ env.TAG }} - ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest + ${{ vars.DOCKER_REGISTRY }}/geolake-drivers:latest - name: Build and push datastore component - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ./datastore file: ./datastore/Dockerfile push: true build-args: | REGISTRY=${{ vars.DOCKER_REGISTRY }} + cache-from: type=gha + cache-to: type=gha,mode=max tags: | ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:${{ env.TAG }} - ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest + ${{ vars.DOCKER_REGISTRY }}/geolake-datastore:latest - name: Build and push api component - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ./api file: ./api/Dockerfile push: true build-args: | REGISTRY=${{ vars.DOCKER_REGISTRY }} + cache-from: type=gha + cache-to: type=gha,mode=max tags: | ${{ vars.DOCKER_REGISTRY }}/geolake-api:${{ env.TAG }} - ${{ vars.DOCKER_REGISTRY }}/geolake-api:latest + ${{ vars.DOCKER_REGISTRY }}/geolake-api:latest - name: Build and push executor component - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ./executor file: ./executor/Dockerfile push: true build-args: | REGISTRY=${{ vars.DOCKER_REGISTRY }} + cache-from: type=gha + cache-to: type=gha,mode=max tags: | ${{ vars.DOCKER_REGISTRY }}/geolake-executor:${{ env.TAG }} - ${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest \ No newline at end of file + ${{ vars.DOCKER_REGISTRY }}/geolake-executor:latest \ No newline at end of file diff --git a/datastore/Dockerfile b/datastore/Dockerfile index 5dc465a..7e051cc 100644 --- a/datastore/Dockerfile +++ b/datastore/Dockerfile @@ -1,10 +1,9 @@ ARG REGISTRY=rg.fr-par.scw.cloud/geolake ARG TAG=latest FROM $REGISTRY/geolake-drivers:$TAG -RUN conda install -c conda-forge --yes --freeze-installed psycopg2 \ - && conda clean -afy + COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir -r /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt COPY ./datastore /app/datastore COPY ./workflow /app/workflow COPY ./dbmanager /app/dbmanager @@ -13,3 +12,4 @@ COPY ./utils /app/utils COPY ./tests /app/tests COPY ./wait-for-it.sh / +WORKDIR / \ No newline at end of file diff --git a/datastore/requirements.txt b/datastore/requirements.txt index 449eb47..0279878 100644 --- a/datastore/requirements.txt +++ b/datastore/requirements.txt @@ -1,2 +1,3 @@ networkx -pydantic<2.0.0 \ No newline at end of file +pydantic<2.0.0 +psycopg2-binary \ No newline at end of file diff --git a/drivers/Dockerfile b/drivers/Dockerfile index 1e8927b..d94bd0c 100644 --- a/drivers/Dockerfile +++ b/drivers/Dockerfile @@ -1,8 +1,8 @@ ARG REGISTRY=rg.fr-par.scw.cloud/geokube +# ARG TAG=v0.2.6b1 ARG TAG=latest FROM $REGISTRY/geokube:$TAG -RUN conda install -c conda-forge --yes --freeze-installed intake=0.6.6 -RUN conda clean -afy + COPY dist/intake_geokube-0.1a0-py3-none-any.whl / RUN pip install /intake_geokube-0.1a0-py3-none-any.whl -RUN rm /intake_geokube-0.1a0-py3-none-any.whl +RUN rm /intake_geokube-0.1a0-py3-none-any.whl \ No newline at end of file From 9049c51d7fec1cfab115b5c5f4a40e07e92d6a4b Mon Sep 17 00:00:00 2001 From: Gabriele Tramonte Date: Wed, 8 May 2024 10:17:11 +0200 Subject: [PATCH 3/3] update workflow for release --- .github/workflows/build_on_release.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_on_release.yaml b/.github/workflows/build_on_release.yaml index 30b2240..4c7d8be 100644 --- a/.github/workflows/build_on_release.yaml +++ b/.github/workflows/build_on_release.yaml @@ -32,7 +32,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Build and push drivers - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ./drivers file: ./drivers/Dockerfile @@ -40,10 +40,12 @@ jobs: build-args: | REGISTRY=${{ vars.GEOKUBE_REGISTRY }} TAG=v0.2a6 + cache-from: type=gha + cache-to: type=gha,mode=max tags: | ${{ vars.GEOLAKE_REGISTRY }}/geolake-drivers:${{ env.RELEASE_TAG }} - name: Build and push datastore component - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ./datastore file: ./datastore/Dockerfile @@ -51,10 +53,12 @@ jobs: build-args: | REGISTRY=${{ vars.GEOLAKE_REGISTRY }} TAG=${{ env.RELEASE_TAG }} + cache-from: type=gha + cache-to: type=gha,mode=max tags: | ${{ vars.GEOLAKE_REGISTRY }}/geolake-datastore:${{ env.RELEASE_TAG }} - name: Build and push api component - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ./api file: ./api/Dockerfile @@ -62,10 +66,12 @@ jobs: build-args: | REGISTRY=${{ vars.GEOLAKE_REGISTRY }} TAG=${{ env.RELEASE_TAG }} + cache-from: type=gha + cache-to: type=gha,mode=max tags: | ${{ vars.GEOLAKE_REGISTRY }}/geolake-api:${{ env.RELEASE_TAG }} - name: Build and push executor component - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: context: ./executor file: ./executor/Dockerfile @@ -73,5 +79,7 @@ jobs: build-args: | REGISTRY=${{ vars.GEOLAKE_REGISTRY }} TAG=${{ env.RELEASE_TAG }} + cache-from: type=gha + cache-to: type=gha,mode=max tags: | ${{ vars.GEOLAKE_REGISTRY }}/geolake-executor:${{ env.RELEASE_TAG }} \ No newline at end of file