fix: CI workflow (#104)

Signed-off-by: a3hadi <[email protected]>
Co-authored-by: Keran Yang <[email protected]>

.github/workflows/build-push.yaml

@@ -7,21 +7,24 @@ on:
       - '*'
 
 jobs:
-  build_push_examples:
-    name: Build and Push Images
+  docker_publish:
+    name: Build, Tag, and Push Images
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        repository: [
-          "map-even-odd"
-          "map-flatmap", "map-forward-message", "map-retry", "map-tickgen"
-          "map-flatmap-stream", "reduce-counter", "reduce-sum", "reduce-stream-counter",
-          "reduce-stream-sum", "session-counter", "sideinput-example", "udf-sideinput-example",
-          "sink-log", "source-simple-source", "mapt-assign-event-time", "mapt-event-time-filter"
+        dockerfile_paths: [
+          "pkg/mapper/examples/even_odd", "pkg/mapper/examples/flatmap", "pkg/mapper/examples/forward_message",
+          "pkg/mapper/examples/retry", "pkg/mapper/examples/tickgen", "pkg/mapstreamer/examples/flatmap_stream",
+          "pkg/reducer/examples/counter", "pkg/reducer/examples/sum", "pkg/reducestreamer/examples/counter",
+          "pkg/reducestreamer/examples/sum", "pkg/sessionreducer/examples/counter", "pkg/sideinput/examples/simple-sideinput/udf",
+          "pkg/sideinput/examples/simple-sideinput", "pkg/sinker/examples/log", "pkg/sourcer/examples/simple_source",
+          "pkg/sourcetransformer/examples/assign_event_time", "pkg/sourcetransformer/examples/event_time_filter"
         ]
 
     steps:
+      - name: Check out repository
+        uses: actions/checkout@v3
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
@@ -32,11 +35,5 @@ jobs:
           registry: quay.io
           username: ${{ secrets.NUMAIO_USERNAME }}
           password: ${{ secrets.NUMAIO_PASSWORD }}
-      - name: Build and tag images
-        run: ./update_examples --build
-      - name: Push images
-        env:
-          IMAGE_PREFIX: "quay.io/numaio/numaflow-go"
-          TAG: "stable"
-        run: |
-          docker push ${{ env.IMAGE_PREFIX }}/${{ matrix.repository }}:${{ env.TAG }}
+      - name: Build, tag, and push images
+        run: ./update_examples.sh --build-push-example ${{ matrix.dockerfile_paths }}

development.md

@@ -7,18 +7,36 @@ This document explains the development process for the Numaflow Go SDK.
 
 The SDK uses local references in the `go.mod` file, i.e. the `github.com/numaproj/numaflow-go` dependency is pointing to your local
 `numaflow-go` directory. Thus, you can automatically test your SDK changes without the need to push to your forked repository or modify the `go.mod` file.
-Simply make your changes, build and push the desired example image, and you are ready to use it in any pipeline.
+Simply make your changes, build and push the desired example image (with an appropriate tag, such as `test`), and you are ready to use it in any pipeline.
+
+Each example directory has a Makefile which can be used to build, tag, and push images. In most cases, the `image-push` target should be used.
+However, `buildx`, which is used to support multiple architectures, does not make a built image available locally. In the case that a developer 
+wants this functionality, they can use the `image` target.
+
+After making changes to the SDK, if you want to build all the example images at once, in the root directory you can run:
+```shell
+./update_examples.sh -bp -t <tag>
+```
+The default tag is `stable`, but it is recommended you specify your own for testing purposes, as the Github Actions CI uses the `stable` tag. Note: do not forget to clean up testing tags
+in the registry, i.e. delete them, once you are done testing.
+
+You can alternatively build a specific example image by running the following in the root directory and providing the path to the Dockerfile (relative to root):
+```shell
+./update_examples.sh -bpe <path> -t <tag>
+```
+This is essentially equivalent to running `make image-push TAG=<tag>` in the example directory itself.
 
 ### Deploying
 
 1. Create a PR for your changes. Once merged, it will trigger a workflow to build and push the images for all the examples, 
 with the tag `stable`. This consistent tag name is used so that the tags in the [E2E test pipelines](https://github.com/numaproj/numaflow/tree/main/test) do not need to be 
 updated each time a new change is made. 
 2. After the changes have been merged it is encouraged to update the dependency management files so that the version 
-displayed in the `go.mod` file reflects the commit SHA of the merged changes. This can be done simply by getting the
-commit SHA of the merged changes and using it with the update script: 
+displayed in the `go.mod` file reflects the commit SHA of the merged changes. This can be done by getting the
+commit SHA of the merged changes and using it with the update script. Alternatively, you can provide the specific version that you would like to update to, or even
+pass in `latest` to fetch the latest version from the remote repository:
 ```shell
-./update_examples -c <commit-sha>
+./update_examples.sh -u <SDK-version | commit-sha | latest>
 ```
 After running the script, create another PR for these changes.
 

pkg/mapper/examples/even_odd/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/even-odd-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/map-even-odd:${TAG}" --platform linux/amd64,linux/arm64 --target even-odd . --push
+
 .PHONY: image
 image: build
-	docker build -t "quay.io/numaio/numaflow-go/map-even-odd:stable" --target even-odd .
+	docker build -t "quay.io/numaio/numaflow-go/map-even-odd:${TAG}" --target even-odd .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-even-odd:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/mapper/examples/flatmap/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/flatmap-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/map-flatmap:${TAG}" --platform linux/amd64,linux/arm64 --target flatmap . --push
+
 .PHONY: image
 image: build
-	docker build -t "quay.io/numaio/numaflow-go/map-flatmap:stable" --target flatmap .
+	docker build -t "quay.io/numaio/numaflow-go/map-flatmap:${TAG}" --target flatmap .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-flatmap:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/mapper/examples/forward_message/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/forward-message-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/map-forward-message:${TAG}" --platform linux/amd64,linux/arm64 --target forward-message . --push
+
 .PHONY: image
 image: build
-	docker build -t "quay.io/numaio/numaflow-go/map-forward-message:stable" --target forward-message .
+	docker build -t "quay.io/numaio/numaflow-go/map-forward-message:${TAG}" --target forward-message .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-forward-message:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/mapper/examples/retry/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/retry main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/map-retry:${TAG}" --platform linux/amd64,linux/arm64 --target retry . --push
+
 .PHONY: image
 image: build
-	docker build -t "quay.io/numaio/numaflow-go/map-retry:stable" --target retry .
+	docker build -t "quay.io/numaio/numaflow-go/map-retry:${TAG}" --target map-retry .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-retry:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/mapper/examples/tickgen/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/tickgen-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/map-tickgen:${TAG}" --platform linux/amd64,linux/arm64 --target tickgen . --push
+
 .PHONY: image
 image: build
-	docker build -t "quay.io/numaio/numaflow-go/map-tickgen:stable" --target tickgen .
+	docker build -t "quay.io/numaio/numaflow-go/map-tickgen:${TAG}" --target tickgen .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-tickgen:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/mapstreamer/examples/flatmap_stream/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/flatmap-stream-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/map-flatmap-stream:${TAG}" --platform linux/amd64,linux/arm64 --target flatmap_stream . --push
+
 .PHONY: image
 image: build
-	docker build -t "quay.io/numaio/numaflow-go/map-flatmap-stream:stable" --target flatmap_stream .
+	docker build -t "quay.io/numaio/numaflow-go/map-flatmap-stream:${TAG}" --target flatmap_stream .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-flatmap-stream:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/reducer/examples/counter/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/reduce-counter:${TAG}" --platform linux/amd64,linux/arm64 --target counter . --push
+
 .PHONY: image
 image: build
-	docker build -t "quay.io/numaio/numaflow-go/reduce-counter:stable" --target counter .
+	docker build -t "quay.io/numaio/numaflow-go/reduce-counter:${TAG}" --target counter .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-counter:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/reducer/examples/sum/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sum-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/reduce-sum:${TAG}" --platform linux/amd64,linux/arm64 --target sum . --push
+
 .PHONY: image
 image: build
-	docker build -t "quay.io/numaio/numaflow-go/reduce-sum:stable" --target sum .
+	docker build -t "quay.io/numaio/numaflow-go/reduce-sum:${TAG}" --target sum .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-sum:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/reducestreamer/examples/counter/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/reduce-stream-counter:${TAG}" --platform linux/amd64,linux/arm64 --target counter . --push
+
 .PHONY: image
 image: build
-	docker build -t "quay.io/numaio/numaflow-go/reduce-stream-counter:stable" --target counter .
+	docker build -t "quay.io/numaio/numaflow-go/reduce-stream-counter:${TAG}" --target counter .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-stream-counter:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/reducestreamer/examples/sum/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sum-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/reduce-stream-sum:${TAG}" --platform linux/amd64,linux/arm64 --target sum . --push
+
 .PHONY: image
 image: build
-	docker build -t "quay.io/numaio/numaflow-go/reduce-stream-sum:stable" --target sum .
+	docker build -t "quay.io/numaio/numaflow-go/reduce-stream-sum:${TAG}" --target sum .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-stream-sum:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/sessionreducer/examples/counter/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/session-counter:${TAG}" --platform linux/amd64,linux/arm64 --target counter . --push
+
 .PHONY: image
 image: build
-	docker build -t "quay.io/numaio/numaflow-go/session-counter:stable" --target counter .
+	docker build -t "quay.io/numaio/numaflow-go/session-counter:${TAG}" --target counter .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/session-counter:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/sideinput/examples/simple-sideinput/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sideinput-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/sideinput-example:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push
+
 .PHONY: image
 image: build
-	docker buildx build -t "quay.io/numaio/numaflow-go/sideinput-example:stable" --platform linux/amd64,linux/arm64 --target sideinput . --push
+	docker build -t "quay.io/numaio/numaflow-go/sideinput-example:${TAG}" --target sideinput .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/sideinput-example:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/sideinput/examples/simple-sideinput/udf/Makefile

@@ -1,13 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/udf-sideinput-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}" --platform linux/amd64,linux/arm64 --target udf-sideinput . --push
+
 .PHONY: image
-#To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command
 image: build
-	docker buildx build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:stable" --platform linux/amd64,linux/arm64 --target udf-sideinput . --push
-
-
+	docker build -t "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}" --target udf-sideinput .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/udf-sideinput-example:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/sinker/examples/log/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/log-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/sink-log:${TAG}" --platform linux/amd64,linux/arm64 --target log . --push
+
 .PHONY: image
-image: build
-	docker build -t "quay.io/numaio/numaflow-go/sink-log:stable" --target log .
+imageimage: build
+	docker build -t "quay.io/numaio/numaflow-go/sink-log:${TAG}" --target log .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/sink-log:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist

pkg/sourcer/examples/simple_source/Makefile

@@ -1,10 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
 .PHONY: build
 build:
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/simple-source-example main.go
 
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/source-simple-source:${TAG}" --platform linux/amd64,linux/arm64 --target simple-source . --push
+
 .PHONY: image
 image: build
-	docker build -t "quay.io/numaio/numaflow-go/source-simple-source:stable" --target simple-source .
+	docker build -t "quay.io/numaio/numaflow-go/source-simple-source:${TAG}" --target simple-source .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/source-simple-source:${TAG}"; fi
 
 clean:
 	-rm -rf ./dist