Merge pull request #35 from earthcube/dev

EC to GleanerIO

Author: fils

.github/workflows/contanerize.yaml

@@ -10,7 +10,7 @@ on:
       - main
       - dev
       - dev_eco
-      - dv_docker
+    #  - dev_dagster142
     tags:
       - "v*.*.*"
 
@@ -22,7 +22,7 @@ defaults:
 
 jobs:
   build:
-    name: Dockerize  Scheduler for Project
+    name: Dockerize  Scheduler
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -33,7 +33,7 @@ jobs:
     steps:
       - name: Set variables
         run: |
-            REGISTRY_IMAGE=nsfearthcube/dagster-${{ matrix.project }}
+            REGISTRY_IMAGE=nsfearthcube/dagster-gleanerio
             echo "REGISTRY_IMAGE=$REGISTRY_IMAGE" >> $GITHUB_ENV
         working-directory:  /
       - name: Checkout Repo
@@ -106,7 +106,102 @@ jobs:
           build-args:
             implnet=${{ matrix.project }}
           #file: ./dagster/implnets/build/Dockerfile
-          file: ./build/Dockerfile_ci
+          file: ./build/Dockerfile_dagster
+          context:  "{{defaultContext}}:dagster/implnets"
+          tags: ${{ steps.meta.outputs.tags }}
+#          tags: nsfearthcube/ec_facets_client:latest
+          labels: ${{ steps.meta.outputs.labels }}
+          #platforms: ${{ matrix.platform }}
+          #outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true
+
+      - name: Image digest
+        run: echo ${{ steps.build.outputs.digest }}
+  build_code:
+    name: Dockerize  Scheduler CODE for Project
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+       # project: [ "eco" ]
+        project: [ "eco", "iow", "oih" ]
+        #platform: ["linux/amd64","linux/arm64"]
+        #platform: ["linux/amd64"] #linux/arm64 issues with building
+    steps:
+      - name: Set variables
+        run: |
+            REGISTRY_IMAGE=nsfearthcube/dagster-gleanerio-${{ matrix.project }}
+            echo "REGISTRY_IMAGE=$REGISTRY_IMAGE" >> $GITHUB_ENV
+        working-directory:  /
+      - name: Checkout Repo
+        uses: actions/checkout@v3
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images:  ${{ env.REGISTRY_IMAGE }}
+          flavor: |
+            latest=true
+          tags: |
+            type=ref,event=tag
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=sha
+
+#      - name: Set up Python 3.10
+#        uses: actions/setup-python@v4
+#        with:
+#          python-version:  '3.10'
+#      - name: Install dependencies
+#        run: |
+#          python -m pip install --upgrade pip
+#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+# long version with lint,
+#        run: |
+#          python -m pip install --upgrade pip
+#          pip install flake8 pytest
+#          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+#          pip install build
+#      - name: Lint with flake8
+#        run: |
+#          # stop the build if there are Python syntax errors or undefined names
+#          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+#          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+#          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+#      - name: Build package
+#        run: python -m build
+#      - name: Build tooling
+#        run: |
+#          cd ./tooling/cfgBuilder/${{ matrix.project }}
+#          python cfgBuilder.py -s https://foo.us/sitemap.xml
+#      - name: Generate
+#        working-directory:  dagster/implnets
+#        run: |
+#          python pygen.py -cf ./configs/${{ matrix.project }}/gleanerconfig.yaml -od /temp/generatedDocker/implnet-${{ matrix.project }}/output  -td ./templates/v1   -d 7
+#      - name: check
+#        working-directory:  dagster/implnets
+#        run: |
+#           cat ./generatedCode/implnet-${{ matrix.project }}/output/ops/implnet_ops_amgeo.py
+
+      - name: Build and push
+        id: build
+        uses: docker/build-push-action@v4
+        with:
+          #context:  ./dagster/implnets
+          # grr https://github.com/docker/build-push-action#git-context
+          #context:  "{{defaultContext}}"
+          push: true
+          build-args:
+            implnet=${{ matrix.project }}
+          #file: ./dagster/implnets/build/Dockerfile
+          file: ./build/Dockerfile_code
           context:  "{{defaultContext}}:dagster/implnets"
           tags: ${{ steps.meta.outputs.tags }}
 #          tags: nsfearthcube/ec_facets_client:latest

NOTES.md

@@ -1,6 +1,29 @@
 # Notes
 
+## Development
 
+At the top level (dagster/implents) you can run 
+
+`dagster dev`
+
+You need to set the environment based on dagster/implnets/deployment/envFile.env
+
+It should run workflows/tasks/tasks
+
+defined in the pyproject.toml
+
+```
+[tool.dagster]
+module_name = "workflows.tasks.tasks"
+```
+
+### testing tasks
+
+cd dagster/implnets/workflows/tasks
+You need to set the environment based on dagster/implnets/deployment/envFile.env
+
+`dagster dev`
+will run just the task, and in editable form, i think.
 
 ## Some articles to review
 

dagster/implnets/Makefile

@@ -47,7 +47,7 @@ oih-clean:
 	cd ./generatedCode/implnet-oih && rm -rf ./output/*
 
 oih-generate:
-	python pygen.py -cf ./configs/oih/gleanerconfig.yaml -od ./generatedCode/implnet-oih/output  -td ./templates/v1   -d 7
+	python pygen.py -cf ./configs/oih/gleanerconfig.yaml -od ./generatedCode/implnet-oih/output  -td ./templates/v1   -d 14
 
 oih-build:
 	podman build  --tag="docker.io/fils/dagster_oih:$(VERSION)"  --build-arg implnet=oih --file=./build/Dockerfile .
@@ -64,7 +64,7 @@ iow-clean:
 	cd ./generatedCode/implnet-iow && rm -rf ./output/*
 
 iow-generate:
-	python pygen.py -cf ./configs/iow/gleanerconfig.yaml -od ./generatedCode/implnet-iow/output  -td ./templates/v1   -d 21
+	python pygen.py -cf ./configs/iow/gleanerconfig.yaml -od ./generatedCode/implnet-iow/output  -td ./templates/v1   -d 27
 
 iow-build:
 	podman build  --tag="docker.io/fils/dagster_iow:$(VERSION)"  --build-arg implnet=iow --file=./build/Dockerfile .

dagster/implnets/README_DEV.md

@@ -0,0 +1,26 @@
+
+Note: Moved the dagster.yaml that defines the container based configuraiton to deployment.
+
+
+To start a dev instance, you can type
+
+set the environment variables in deploymnet/envFile.env 
+   * 
+   * if using jetbrains/pycharm, use a shell script, and add the env variables.
+
+
+`dagster dev --workspace workspace_dev.yaml`
+
+
+You can go to deployment, and get the code reloaded by reloading the definitions.
+
+## Using local deployment and containers
+
+in deploy, you can use the dagster_local.sh
+
+when combined with the proper env, it will use the workflows.yaml in config/$PROJECT/workflows.yam
+a
+In prep for using a service that can mount the gleaner/nabu configs, the local 
+also creates 'file' configs using   config/$PROJECT/gleanerconfig.yaml and  config/$PROJECT/nabuconfig.yaml
+
+

dagster/implnets/build/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-slim
+FROM python:3.11-slim
 
 # Read the ARG implnet to set who to build for.
 

dagster/implnets/build/Dockerfile_ci

@@ -5,7 +5,8 @@ FROM python:3.11-slim
 # docker buildandpush pulls the repo, so we need to put the code at a different location
 # this fails becaus the dagster/implnets files are not in the docker
 ARG implnet=eco
-RUN mkdir -p /usr/src/app/
+RUN mkdir -p /usr/src/app/project/${implnet}
+RUN mkdir -p /usr/src/app/workflows
 
 RUN pip install --upgrade pip
 ## this is a base for the project. Build this 'layer' first
@@ -16,20 +17,22 @@ RUN pip install -r requirements.txt
 COPY . scheduler
 COPY ./configs/${implnet}/gleanerconfig.yaml scheduler/gleanerconfig.yaml
 
-COPY dagster.yaml /usr/src/app/
+COPY ./deployment/dagster.yaml /usr/src/app/
 
 WORKDIR scheduler
 
 
 RUN pip install -r requirements.txt
-RUN python pygen.py -cf ./gleanerconfig.yaml -od /usr/src/app/  -td ./templates/v1   -d 7
+RUN python pygen.py -cf ./gleanerconfig.yaml -od /usr/src/app/project/${implnet}  -td ./templates/v1   -d 7
 
 
+COPY ./workflows/ /usr/src/app/workflows
+
 
 
 # Change working directory
 WORKDIR /usr/src/app
 ENV DAGSTER_HOME=/usr/src/app
 
 
-CMD ["dagit", "-w", "workspace.yaml", "-h", "0.0.0.0", "-p", "3000"]
+CMD ["dagster-webserver", "-w", "./project/${implnet}/workspace.yaml", "-h", "0.0.0.0", "-p", "3000"]

dagster/implnets/build/Dockerfile_code

@@ -0,0 +1,40 @@
+FROM python:3.11-slim
+
+# we may want to get an unreleased version of code, so this is needed
+RUN apt-get update && apt-get install -y git
+# Read the ARG implnet to set who to build for.
+
+# docker buildandpush pulls the repo, so we need to put the code at a different location
+# this fails becaus the dagster/implnets files are not in the docker
+ARG implnet=eco
+RUN mkdir -p /usr/src/app/project/${implnet}
+RUN mkdir -p /usr/src/app/workflows
+
+RUN pip install --upgrade pip
+## this is a base for the project. Build this 'layer' first
+COPY ./requirements_code.txt requirements.txt
+RUN pip install -r requirements.txt
+
+# this add the code
+COPY . scheduler
+COPY ./configs/${implnet}/gleanerconfig.yaml scheduler/gleanerconfig.yaml
+
+COPY ./deployment/dagster.yaml /usr/src/app/
+
+WORKDIR scheduler
+
+
+RUN pip install -r requirements.txt
+RUN python pygen.py -cf ./gleanerconfig.yaml -od /usr/src/app/project/${implnet}  -td ./templates/v1   -d 7
+
+
+COPY ./workflows/ /usr/src/app/workflows
+
+
+
+# Change working directory
+WORKDIR /usr/src/app
+ENV DAGSTER_HOME=/usr/src/app
+
+
+CMD [ "dagster", "api","grpc", "-h", "0.0.0.0", "-p", "4000", "--python-file", "/usr/src/app/project/${PROJECT:-eco}/repositories/repository.py", "-d", "/usr/src/app/project/${PROJECT:-eco}/"]

dagster/implnets/build/Dockerfile_dagster

@@ -0,0 +1,36 @@
+FROM python:3.11-slim
+
+# Read the ARG implnet to set who to build for.
+
+# docker buildandpush pulls the repo, so we need to put the code at a different location
+# this fails becaus the dagster/implnets files are not in the docker
+
+
+RUN pip install --upgrade pip
+## this is a base for the project. Build this 'layer' first
+COPY ./requirements_dagster.txt requirements.txt
+RUN pip install -r requirements.txt
+
+## this add the code
+#COPY . scheduler
+#COPY ./configs/${implnet}/gleanerconfig.yaml scheduler/gleanerconfig.yaml
+#
+COPY ./deployment/dagster.yaml /usr/src/app/
+#
+#WORKDIR scheduler
+#
+#
+#RUN pip install -r requirements.txt
+#RUN python pygen.py -cf ./gleanerconfig.yaml -od /usr/src/app/project/${implnet}  -td ./templates/v1   -d 7
+#
+#
+#COPY ./workflows/ /usr/src/app/workflows
+
+
+
+# Change working directory
+WORKDIR /usr/src/app
+ENV DAGSTER_HOME=/usr/src/app
+
+
+CMD ["dagster-webserver", "-w", "workspace.yaml", "-h", "0.0.0.0", "-p", "3000"]

dagster/implnets/build/Dockerfile_local

@@ -3,7 +3,8 @@ FROM python:3.11-slim
 # for local, code is compiled. to generatedCode/implnet-PROJECT
 
 ARG implnet=eco
-RUN mkdir -p /usr/src/app/output
+RUN mkdir -p /usr/src/app/project
+RUN mkdir -p /usr/src/app/workflows
 
 COPY ./requirements.txt .
 
@@ -16,12 +17,12 @@ COPY . scheduler
 COPY ./configs/${implnet}/gleanerconfig.yaml scheduler/gleanerconfig.yaml
 
 #COPY ./dagster.yaml /usr/src/app/output/dagster.yaml
-COPY ./dagster.yaml /usr/src/app/dagster.yaml
+COPY ./deployment/dagster.yaml /usr/src/app/
 # Change working directory
 
 
 WORKDIR /usr/src/app
 ENV DAGSTER_HOME=/usr/src/app
 
 
-CMD ["dagit", "-w", "./output/workspace.yaml", "-h", "0.0.0.0", "-p", "3000"]
+CMD ["dagster-webserver", "-w", "./project/${implnet}/workspace.yaml", "-h", "0.0.0.0", "-p", "3000"]

dagster/implnets/build/Dockerfile_multistage

@@ -0,0 +1,48 @@
+FROM python:3.11-slim as base
+
+# Read the ARG implnet to set who to build for.
+
+# docker buildandpush pulls the repo, so we need to put the code at a different location
+# this fails becaus the dagster/implnets files are not in the docker
+ARG implnet=eco
+RUN mkdir -p /usr/src/app/project/${implnet}
+RUN mkdir -p /usr/src/app/workflows
+
+RUN pip install --upgrade pip
+## this is a base for the project. Build this 'layer' first
+COPY ./requirements.txt .
+RUN pip install -r requirements.txt
+
+
+# Change working directory
+WORKDIR /usr/src/app
+ENV DAGSTER_HOME=/usr/src/app
+
+
+CMD ["dagster-webserver", "-w", "./project/${implnet}/workspace.yaml", "-h", "0.0.0.0", "-p", "3000"]
+
+FROM base as dagster
+WORKDIR /usr/src/app
+ENV DAGSTER_HOME=/usr/src/app
+
+
+CMD ["dagster-webserver", "-w", "./project/${implnet}/workspace.yaml", "-h", "0.0.0.0", "-p", "3000"]
+
+
+FROM base AS dagstercode
+# this add the code
+COPY . scheduler
+COPY ./configs/${implnet}/gleanerconfig.yaml scheduler/gleanerconfig.yaml
+
+COPY ./deployment/dagster.yaml /usr/src/app/
+
+WORKDIR scheduler
+
+
+RUN pip install -r requirements.txt
+RUN python pygen.py -cf ./gleanerconfig.yaml -od /usr/src/app/project/${implnet}  -td ./templates/v1   -d 7
+
+
+COPY ./workflows/ /usr/src/app/workflows
+
+