From 3e766e91e3c0004a1eb58e3add9956ef01c8cbea Mon Sep 17 00:00:00 2001 From: Abdullah Yildirim Date: Tue, 12 Mar 2024 10:39:52 -0400 Subject: [PATCH] fix: CI workflow (#104) Signed-off-by: a3hadi Co-authored-by: Keran Yang Signed-off-by: a3hadi --- .github/workflows/build-push.yaml | 29 +++--- development.md | 26 +++++- pkg/mapper/examples/even_odd/Makefile | 10 ++- pkg/mapper/examples/flatmap/Makefile | 10 ++- pkg/mapper/examples/forward_message/Makefile | 10 ++- pkg/mapper/examples/retry/Makefile | 10 ++- pkg/mapper/examples/tickgen/Makefile | 10 ++- .../examples/flatmap_stream/Makefile | 10 ++- pkg/reducer/examples/counter/Makefile | 10 ++- pkg/reducer/examples/sum/Makefile | 10 ++- pkg/reducestreamer/examples/counter/Makefile | 10 ++- pkg/reducestreamer/examples/sum/Makefile | 10 ++- pkg/sessionreducer/examples/counter/Makefile | 10 ++- .../examples/simple-sideinput/Makefile | 10 ++- .../examples/simple-sideinput/udf/Makefile | 13 ++- pkg/sinker/examples/log/Makefile | 12 ++- pkg/sourcer/examples/simple_source/Makefile | 10 ++- .../examples/assign_event_time/Makefile | 10 ++- .../examples/event_time_filter/Makefile | 10 ++- update_examples.sh | 88 ++++++++++++++----- 20 files changed, 257 insertions(+), 61 deletions(-) diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml index 7637f73a..5c331de6 100644 --- a/.github/workflows/build-push.yaml +++ b/.github/workflows/build-push.yaml @@ -7,21 +7,24 @@ on: - '*' jobs: - build_push_examples: - name: Build and Push Images + docker_publish: + name: Build, Tag, and Push Images runs-on: ubuntu-latest strategy: matrix: - repository: [ - "map-even-odd" - "map-flatmap", "map-forward-message", "map-retry", "map-tickgen" - "map-flatmap-stream", "reduce-counter", "reduce-sum", "reduce-stream-counter", - "reduce-stream-sum", "session-counter", "sideinput-example", "udf-sideinput-example", - "sink-log", "source-simple-source", "mapt-assign-event-time", "mapt-event-time-filter" + dockerfile_paths: [ + "pkg/mapper/examples/even_odd", "pkg/mapper/examples/flatmap", "pkg/mapper/examples/forward_message", + "pkg/mapper/examples/retry", "pkg/mapper/examples/tickgen", "pkg/mapstreamer/examples/flatmap_stream", + "pkg/reducer/examples/counter", "pkg/reducer/examples/sum", "pkg/reducestreamer/examples/counter", + "pkg/reducestreamer/examples/sum", "pkg/sessionreducer/examples/counter", "pkg/sideinput/examples/simple-sideinput/udf", + "pkg/sideinput/examples/simple-sideinput", "pkg/sinker/examples/log", "pkg/sourcer/examples/simple_source", + "pkg/sourcetransformer/examples/assign_event_time", "pkg/sourcetransformer/examples/event_time_filter" ] steps: + - name: Check out repository + uses: actions/checkout@v3 - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx @@ -32,11 +35,5 @@ jobs: registry: quay.io username: ${{ secrets.NUMAIO_USERNAME }} password: ${{ secrets.NUMAIO_PASSWORD }} - - name: Build and tag images - run: ./update_examples --build - - name: Push images - env: - IMAGE_PREFIX: "quay.io/numaio/numaflow-go" - TAG: "stable" - run: | - docker push ${{ env.IMAGE_PREFIX }}/${{ matrix.repository }}:${{ env.TAG }} + - name: Build, tag, and push images + run: ./update_examples.sh --build-push-example ${{ matrix.dockerfile_paths }} diff --git a/development.md b/development.md index a8cae1ab..c20d1f3c 100644 --- a/development.md +++ b/development.md @@ -7,7 +7,24 @@ This document explains the development process for the Numaflow Go SDK. The SDK uses local references in the `go.mod` file, i.e. the `github.com/numaproj/numaflow-go` dependency is pointing to your local `numaflow-go` directory. Thus, you can automatically test your SDK changes without the need to push to your forked repository or modify the `go.mod` file. -Simply make your changes, build and push the desired example image, and you are ready to use it in any pipeline. +Simply make your changes, build and push the desired example image (with an appropriate tag, such as `test`), and you are ready to use it in any pipeline. + +Each example directory has a Makefile which can be used to build, tag, and push images. In most cases, the `image-push` target should be used. +However, `buildx`, which is used to support multiple architectures, does not make a built image available locally. In the case that a developer +wants this functionality, they can use the `image` target. + +After making changes to the SDK, if you want to build all the example images at once, in the root directory you can run: +```shell +./update_examples.sh -bp -t +``` +The default tag is `stable`, but it is recommended you specify your own for testing purposes, as the Github Actions CI uses the `stable` tag. Note: do not forget to clean up testing tags +in the registry, i.e. delete them, once you are done testing. + +You can alternatively build a specific example image by running the following in the root directory and providing the path to the Dockerfile (relative to root): +```shell +./update_examples.sh -bpe -t +``` +This is essentially equivalent to running `make image-push TAG=` in the example directory itself. ### Deploying @@ -15,10 +32,11 @@ Simply make your changes, build and push the desired example image, and you are with the tag `stable`. This consistent tag name is used so that the tags in the [E2E test pipelines](https://github.com/numaproj/numaflow/tree/main/test) do not need to be updated each time a new change is made. 2. After the changes have been merged it is encouraged to update the dependency management files so that the version -displayed in the `go.mod` file reflects the commit SHA of the merged changes. This can be done simply by getting the -commit SHA of the merged changes and using it with the update script: +displayed in the `go.mod` file reflects the commit SHA of the merged changes. This can be done by getting the +commit SHA of the merged changes and using it with the update script. Alternatively, you can provide the specific version that you would like to update to, or even +pass in `latest` to fetch the latest version from the remote repository: ```shell -./update_examples -c +./update_examples.sh -u ``` After running the script, create another PR for these changes. diff --git a/pkg/mapper/examples/even_odd/Makefile b/pkg/mapper/examples/even_odd/Makefile index 87609b7d..048194ab 100644 --- a/pkg/mapper/examples/even_odd/Makefile +++ b/pkg/mapper/examples/even_odd/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/even-odd-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/map-even-odd:${TAG}" --platform linux/amd64,linux/arm64 --target even-odd . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/map-even-odd:stable" --target even-odd . + docker build -t "quay.io/numaio/numaflow-go/map-even-odd:${TAG}" --target even-odd . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-even-odd:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/mapper/examples/flatmap/Makefile b/pkg/mapper/examples/flatmap/Makefile index 87a75e8a..c7cebf2c 100644 --- a/pkg/mapper/examples/flatmap/Makefile +++ b/pkg/mapper/examples/flatmap/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/flatmap-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/map-flatmap:${TAG}" --platform linux/amd64,linux/arm64 --target flatmap . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/map-flatmap:stable" --target flatmap . + docker build -t "quay.io/numaio/numaflow-go/map-flatmap:${TAG}" --target flatmap . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-flatmap:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/mapper/examples/forward_message/Makefile b/pkg/mapper/examples/forward_message/Makefile index 67ee2531..8e82c0d0 100644 --- a/pkg/mapper/examples/forward_message/Makefile +++ b/pkg/mapper/examples/forward_message/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/forward-message-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/map-forward-message:${TAG}" --platform linux/amd64,linux/arm64 --target forward-message . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/map-forward-message:stable" --target forward-message . + docker build -t "quay.io/numaio/numaflow-go/map-forward-message:${TAG}" --target forward-message . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-forward-message:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/mapper/examples/retry/Makefile b/pkg/mapper/examples/retry/Makefile index bdd63e02..ab8a5a70 100644 --- a/pkg/mapper/examples/retry/Makefile +++ b/pkg/mapper/examples/retry/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/retry main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/map-retry:${TAG}" --platform linux/amd64,linux/arm64 --target retry . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/map-retry:stable" --target retry . + docker build -t "quay.io/numaio/numaflow-go/map-retry:${TAG}" --target map-retry . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-retry:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/mapper/examples/tickgen/Makefile b/pkg/mapper/examples/tickgen/Makefile index 34580b90..78c78fbd 100644 --- a/pkg/mapper/examples/tickgen/Makefile +++ b/pkg/mapper/examples/tickgen/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/tickgen-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/map-tickgen:${TAG}" --platform linux/amd64,linux/arm64 --target tickgen . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/map-tickgen:stable" --target tickgen . + docker build -t "quay.io/numaio/numaflow-go/map-tickgen:${TAG}" --target tickgen . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-tickgen:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/mapstreamer/examples/flatmap_stream/Makefile b/pkg/mapstreamer/examples/flatmap_stream/Makefile index 8f7b8966..18945ac4 100644 --- a/pkg/mapstreamer/examples/flatmap_stream/Makefile +++ b/pkg/mapstreamer/examples/flatmap_stream/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/flatmap-stream-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/map-flatmap-stream:${TAG}" --platform linux/amd64,linux/arm64 --target flatmap_stream . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/map-flatmap-stream:stable" --target flatmap_stream . + docker build -t "quay.io/numaio/numaflow-go/map-flatmap-stream:${TAG}" --target flatmap_stream . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-flatmap-stream:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/reducer/examples/counter/Makefile b/pkg/reducer/examples/counter/Makefile index b04f0ace..a30a6e61 100644 --- a/pkg/reducer/examples/counter/Makefile +++ b/pkg/reducer/examples/counter/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/reduce-counter:${TAG}" --platform linux/amd64,linux/arm64 --target counter . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/reduce-counter:stable" --target counter . + docker build -t "quay.io/numaio/numaflow-go/reduce-counter:${TAG}" --target counter . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-counter:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/reducer/examples/sum/Makefile b/pkg/reducer/examples/sum/Makefile index f3e9e8d1..5e8b01bc 100644 --- a/pkg/reducer/examples/sum/Makefile +++ b/pkg/reducer/examples/sum/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sum-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/reduce-sum:${TAG}" --platform linux/amd64,linux/arm64 --target sum . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/reduce-sum:stable" --target sum . + docker build -t "quay.io/numaio/numaflow-go/reduce-sum:${TAG}" --target sum . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-sum:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/reducestreamer/examples/counter/Makefile b/pkg/reducestreamer/examples/counter/Makefile index bc6e4c4d..471e33c0 100644 --- a/pkg/reducestreamer/examples/counter/Makefile +++ b/pkg/reducestreamer/examples/counter/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/reduce-stream-counter:${TAG}" --platform linux/amd64,linux/arm64 --target counter . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/reduce-stream-counter:stable" --target counter . + docker build -t "quay.io/numaio/numaflow-go/reduce-stream-counter:${TAG}" --target counter . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-stream-counter:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/reducestreamer/examples/sum/Makefile b/pkg/reducestreamer/examples/sum/Makefile index 9cf335b6..150e291b 100644 --- a/pkg/reducestreamer/examples/sum/Makefile +++ b/pkg/reducestreamer/examples/sum/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sum-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/reduce-stream-sum:${TAG}" --platform linux/amd64,linux/arm64 --target sum . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/reduce-stream-sum:stable" --target sum . + docker build -t "quay.io/numaio/numaflow-go/reduce-stream-sum:${TAG}" --target sum . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-stream-sum:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sessionreducer/examples/counter/Makefile b/pkg/sessionreducer/examples/counter/Makefile index 19c37944..360b416c 100644 --- a/pkg/sessionreducer/examples/counter/Makefile +++ b/pkg/sessionreducer/examples/counter/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/session-counter:${TAG}" --platform linux/amd64,linux/arm64 --target counter . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/session-counter:stable" --target counter . + docker build -t "quay.io/numaio/numaflow-go/session-counter:${TAG}" --target counter . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/session-counter:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sideinput/examples/simple-sideinput/Makefile b/pkg/sideinput/examples/simple-sideinput/Makefile index 21426717..1f6173c9 100644 --- a/pkg/sideinput/examples/simple-sideinput/Makefile +++ b/pkg/sideinput/examples/simple-sideinput/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sideinput-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/sideinput-example:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push + .PHONY: image image: build - docker buildx build -t "quay.io/numaio/numaflow-go/sideinput-example:stable" --platform linux/amd64,linux/arm64 --target sideinput . --push + docker build -t "quay.io/numaio/numaflow-go/sideinput-example:${TAG}" --target sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/sideinput-example:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sideinput/examples/simple-sideinput/udf/Makefile b/pkg/sideinput/examples/simple-sideinput/udf/Makefile index eb1456c0..22c3ba60 100644 --- a/pkg/sideinput/examples/simple-sideinput/udf/Makefile +++ b/pkg/sideinput/examples/simple-sideinput/udf/Makefile @@ -1,13 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/udf-sideinput-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}" --platform linux/amd64,linux/arm64 --target udf-sideinput . --push + .PHONY: image -#To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command image: build - docker buildx build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:stable" --platform linux/amd64,linux/arm64 --target udf-sideinput . --push - - + docker build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}" --target udf-sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sinker/examples/log/Makefile b/pkg/sinker/examples/log/Makefile index 37e42e94..0850b8dc 100644 --- a/pkg/sinker/examples/log/Makefile +++ b/pkg/sinker/examples/log/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/log-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/sink-log:${TAG}" --platform linux/amd64,linux/arm64 --target log . --push + .PHONY: image -image: build - docker build -t "quay.io/numaio/numaflow-go/sink-log:stable" --target log . +imageimage: build + docker build -t "quay.io/numaio/numaflow-go/sink-log:${TAG}" --target log . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/sink-log:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sourcer/examples/simple_source/Makefile b/pkg/sourcer/examples/simple_source/Makefile index 61c35287..47d803e5 100644 --- a/pkg/sourcer/examples/simple_source/Makefile +++ b/pkg/sourcer/examples/simple_source/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/simple-source-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/source-simple-source:${TAG}" --platform linux/amd64,linux/arm64 --target simple-source . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/source-simple-source:stable" --target simple-source . + docker build -t "quay.io/numaio/numaflow-go/source-simple-source:${TAG}" --target simple-source . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/source-simple-source:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sourcetransformer/examples/assign_event_time/Makefile b/pkg/sourcetransformer/examples/assign_event_time/Makefile index 34a183ef..969afea4 100644 --- a/pkg/sourcetransformer/examples/assign_event_time/Makefile +++ b/pkg/sourcetransformer/examples/assign_event_time/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/assign-event-time-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/mapt-assign-event-time:${TAG}" --platform linux/amd64,linux/arm64 --target assign-event-time . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/mapt-assign-event-time:stable" --target assign-event-time . + docker build -t "quay.io/numaio/numaflow-go/mapt-assign-event-time:${TAG}" --target assign-event-time . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/mapt-assign-event-time:${TAG}"; fi clean: -rm -rf ./dist diff --git a/pkg/sourcetransformer/examples/event_time_filter/Makefile b/pkg/sourcetransformer/examples/event_time_filter/Makefile index b94f3144..b8e2c21f 100644 --- a/pkg/sourcetransformer/examples/event_time_filter/Makefile +++ b/pkg/sourcetransformer/examples/event_time_filter/Makefile @@ -1,10 +1,18 @@ +TAG ?= stable +PUSH ?= false + .PHONY: build build: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/event-time-filter-example main.go +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/mapt-event-time-filter:${TAG}" --platform linux/amd64,linux/arm64 --target event-time-filter . --push + .PHONY: image image: build - docker build -t "quay.io/numaio/numaflow-go/mapt-event-time-filter:stable" --target event-time-filter . + docker build -t "quay.io/numaio/numaflow-go/mapt-event-time-filter:${TAG}" --target event-time-filter . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/mapt-event-time-filter:${TAG}"; fi clean: -rm -rf ./dist diff --git a/update_examples.sh b/update_examples.sh index 09b9fdbe..5051b0f7 100755 --- a/update_examples.sh +++ b/update_examples.sh @@ -1,10 +1,12 @@ #!/bin/bash function show_help () { - echo "Usage: $0 [-h|--help] (-b|--build | -c|--commit-sha )" - echo " -h, --help Display help message and exit" - echo " -b, --build Build the docker images of all the examples" - echo " -c, --commit-sha Update all of the examples to depend on the numaflow-go version with the specified SHA or version" + echo "Usage: $0 [-h|--help | -t|--tag ] (-bp|--build-push | -bpe|--build-push-example | -u|--update )" + echo " -h, --help Display help message and exit" + echo " -bp, --build-push Build the Dockerfiles of all the examples and push them to the quay.io registry" + echo " -bpe, --build-push-example Build the Dockerfile of the given example directory path, and push it to the quay.io registry" + echo " -t, --tag To be optionally used with -bpe or -bp. Specify the tag to build with. Default tag: stable" + echo " -u, --update Update all of the examples to depend on the numaflow-go version with the specified SHA or version" } function traverse_examples () { @@ -32,9 +34,13 @@ if [ $# -eq 0 ]; then fi usingHelp=0 -usingBuild=0 -usingSHA=0 -commit_sha="" +usingBuildPush=0 +usingBuildPushExample=0 +usingVersion=0 +usingTag=0 +version="" +directoryPath="" +tag="stable" function handle_options () { while [ $# -gt 0 ]; do @@ -42,18 +48,40 @@ function handle_options () { -h | --help) usingHelp=1 ;; - -b | --build) - usingBuild=1 + -bp | --build-push) + usingBuildPush=1 ;; - -c | --commit-sha) - usingSHA=1 + -bpe | --build-push-example) + if [ -z "$2" ]; then + echo "Directory path not specified." >&2 + show_help + exit 1 + fi + + usingBuildPushExample=1 + directoryPath=$2 + shift + ;; + -t | --tag) + if [ -z "$2" ]; then + echo "Tag not specified." >&2 + show_help + exit 1 + fi + + usingTag=1 + tag=$2 + shift + ;; + -u | --update) if [ -z "$2" ]; then echo "Commit SHA or version not specified." >&2 show_help exit 1 fi - commit_sha=$2 + usingVersion=1 + version=$2 shift ;; *) @@ -68,20 +96,40 @@ function handle_options () { handle_options "$@" -if (( usingBuild + usingSHA + usingHelp > 1 )); then - echo "Only one of '-h', '-b', '-c' is allowed at a time" >&2 +if (( usingBuildPush + usingBuildPushExample + usingVersion + usingHelp > 1 )); then + echo "Only one of '-h', '-bp', '-bpe', or, '-u' is allowed at a time" >&2 show_help exit 1 fi -if [ -n "$commit_sha" ]; then - echo "Will update to: $commit_sha" +if (( usingTag + usingVersion + usingHelp > 1 || (usingTag && usingBuildPush + usingBuildPushExample == 0) )); then + echo "Can only use -t with -bp or -bpe" >&2 + show_help + exit 1 +fi + +if [ -n "$version" ]; then + echo "Will update to: $version" +fi + +if [ -n "$directoryPath" ]; then + echo "Dockerfile path to use: $directoryPath" +fi + +if [ -n "$tag" ] && (( ! usingVersion )) && (( ! usingHelp )); then + echo "Using tag: $tag" fi -if (( usingBuild )); then - traverse_examples "make image" -elif (( usingSHA )); then - traverse_examples "go get github.com/numaproj/numaflow-go@$commit_sha" "go mod tidy" +if (( usingBuildPush )); then + traverse_examples "make image-push TAG=$tag" +elif (( usingBuildPushExample )); then + cd "./$directoryPath" || exit + if ! make image-push TAG="$tag"; then + echo "Error: failed to run make image-push in $directoryPath" >&2 + exit 1 + fi +elif (( usingVersion )); then + traverse_examples "go get github.com/numaproj/numaflow-go@$version" "go mod tidy" elif (( usingHelp )); then show_help fi