From 1b380fc492cef9c5d2ef12e958d760d8cb561f60 Mon Sep 17 00:00:00 2001 From: a3hadi Date: Tue, 23 Apr 2024 11:02:05 -0400 Subject: [PATCH] modify makefiles and dockerfiles Signed-off-by: a3hadi --- .github/workflows/build-push.yaml | 8 +++--- .../examples/map-side-input/Makefile | 25 ------------------- .../examples/map-side-input/udf/Makefile | 25 ------------------- .../examples/map_sideinput/Dockerfile | 20 +++++++++++++++ pkg/sideinput/examples/map_sideinput/Makefile | 18 +++++++++++++ .../README.md | 6 ++--- .../{map-side-input => map_sideinput}/go.mod | 2 +- .../{map-side-input => map_sideinput}/go.sum | 0 .../{map-side-input => map_sideinput}/main.go | 0 .../udf/Dockerfile | 2 +- .../examples/map_sideinput/udf/Makefile | 18 +++++++++++++ .../udf/go.mod | 2 ++ .../udf/go.sum | 2 -- .../udf/main.go | 0 .../examples/reduce-side-input/Dockerfile | 20 --------------- .../examples/reduce-side-input/Makefile | 25 ------------------- .../examples/reduce-side-input/udf/Makefile | 25 ------------------- .../Dockerfile | 10 ++++---- .../examples/reduce_sideinput/Makefile | 18 +++++++++++++ .../README.md | 6 ++--- .../go.mod | 2 +- .../go.sum | 0 .../main.go | 0 .../udf/Dockerfile | 4 +-- .../examples/reduce_sideinput/udf/Makefile | 18 +++++++++++++ .../udf/go.mod | 2 ++ .../udf/go.sum | 2 -- .../udf/main.go | 2 +- .../examples/sideInput-function/Makefile | 23 ----------------- .../Dockerfile | 8 +++--- .../examples/sideinput_function/Makefile | 18 +++++++++++++ .../README.md} | 4 +-- .../go.mod | 2 +- .../go.sum | 0 .../main.go | 0 .../simple-source-with-sideinput/Makefile | 23 ----------------- .../Dockerfile | 0 .../Makefile | 0 .../README.md | 4 +-- .../go.mod | 2 +- .../go.sum | 0 .../main.go | 0 .../pipeline.yaml | 0 .../udf/Dockerfile | 0 .../udf/Makefile | 0 .../udf/go.mod | 0 .../udf/go.sum | 0 .../udf/main.go | 0 .../Dockerfile | 2 +- .../simple_source_with_sideinput/Makefile | 18 +++++++++++++ .../README.md} | 0 .../go.mod | 2 +- .../go.sum | 0 .../impl/simple_source_sideinput.go | 0 .../impl/simple_source_sideinput_test.go | 0 .../main.go | 0 .../examples/sink-side-input/Makefile | 23 ----------------- .../examples/sink_sideinput}/Dockerfile | 10 ++++---- .../examples/sink_sideinput/Makefile | 18 +++++++++++++ .../README.md | 7 +++--- .../go.mod | 2 +- .../go.sum | 0 .../main.go | 0 .../examples/redis-e2e-test-sink/Makefile | 10 -------- .../examples/redis-sink}/Dockerfile | 8 +++--- pkg/sinker/examples/redis-sink/Makefile | 18 +++++++++++++ .../README.md | 0 .../go.mod | 0 .../go.sum | 0 .../main.go | 0 70 files changed, 212 insertions(+), 252 deletions(-) delete mode 100644 pkg/sideinput/examples/map-side-input/Makefile delete mode 100644 pkg/sideinput/examples/map-side-input/udf/Makefile create mode 100644 pkg/sideinput/examples/map_sideinput/Dockerfile create mode 100644 pkg/sideinput/examples/map_sideinput/Makefile rename pkg/sideinput/examples/{map-side-input => map_sideinput}/README.md (80%) rename pkg/sideinput/examples/{map-side-input => map_sideinput}/go.mod (95%) rename pkg/sideinput/examples/{map-side-input => map_sideinput}/go.sum (100%) rename pkg/sideinput/examples/{map-side-input => map_sideinput}/main.go (100%) rename pkg/sideinput/examples/{map-side-input => map_sideinput}/udf/Dockerfile (96%) create mode 100644 pkg/sideinput/examples/map_sideinput/udf/Makefile rename pkg/sideinput/examples/{map-side-input => map_sideinput}/udf/go.mod (88%) rename pkg/sideinput/examples/{map-side-input => map_sideinput}/udf/go.sum (93%) rename pkg/sideinput/examples/{map-side-input => map_sideinput}/udf/main.go (100%) delete mode 100644 pkg/sideinput/examples/reduce-side-input/Dockerfile delete mode 100644 pkg/sideinput/examples/reduce-side-input/Makefile delete mode 100644 pkg/sideinput/examples/reduce-side-input/udf/Makefile rename pkg/sideinput/examples/{map-side-input => reduce_sideinput}/Dockerfile (72%) create mode 100644 pkg/sideinput/examples/reduce_sideinput/Makefile rename pkg/sideinput/examples/{reduce-side-input => reduce_sideinput}/README.md (80%) rename pkg/sideinput/examples/{reduce-side-input => reduce_sideinput}/go.mod (94%) rename pkg/sideinput/examples/{reduce-side-input => reduce_sideinput}/go.sum (100%) rename pkg/sideinput/examples/{reduce-side-input => reduce_sideinput}/main.go (100%) rename pkg/sideinput/examples/{reduce-side-input => reduce_sideinput}/udf/Dockerfile (92%) create mode 100644 pkg/sideinput/examples/reduce_sideinput/udf/Makefile rename pkg/sideinput/examples/{reduce-side-input => reduce_sideinput}/udf/go.mod (89%) rename pkg/sideinput/examples/{reduce-side-input => reduce_sideinput}/udf/go.sum (93%) rename pkg/sideinput/examples/{reduce-side-input => reduce_sideinput}/udf/main.go (98%) delete mode 100644 pkg/sideinput/examples/sideInput-function/Makefile rename pkg/sideinput/examples/{sideInput-function => sideinput_function}/Dockerfile (78%) create mode 100644 pkg/sideinput/examples/sideinput_function/Makefile rename pkg/sideinput/examples/{sideInput-function/readme.md => sideinput_function/README.md} (93%) rename pkg/sideinput/examples/{sideInput-function => sideinput_function}/go.mod (94%) rename pkg/sideinput/examples/{sideInput-function => sideinput_function}/go.sum (100%) rename pkg/sideinput/examples/{sideInput-function => sideinput_function}/main.go (100%) delete mode 100644 pkg/sideinput/examples/simple-source-with-sideinput/Makefile rename pkg/sideinput/examples/{simple-sideinput => simple_sideinput}/Dockerfile (100%) rename pkg/sideinput/examples/{simple-sideinput => simple_sideinput}/Makefile (100%) rename pkg/sideinput/examples/{simple-sideinput => simple_sideinput}/README.md (94%) rename pkg/sideinput/examples/{simple-sideinput => simple_sideinput}/go.mod (95%) rename pkg/sideinput/examples/{simple-sideinput => simple_sideinput}/go.sum (100%) rename pkg/sideinput/examples/{simple-sideinput => simple_sideinput}/main.go (100%) rename pkg/sideinput/examples/{simple-sideinput => simple_sideinput}/pipeline.yaml (100%) rename pkg/sideinput/examples/{simple-sideinput => simple_sideinput}/udf/Dockerfile (100%) rename pkg/sideinput/examples/{simple-sideinput => simple_sideinput}/udf/Makefile (100%) rename pkg/sideinput/examples/{simple-sideinput => simple_sideinput}/udf/go.mod (100%) rename pkg/sideinput/examples/{simple-sideinput => simple_sideinput}/udf/go.sum (100%) rename pkg/sideinput/examples/{simple-sideinput => simple_sideinput}/udf/main.go (100%) rename pkg/sideinput/examples/{simple-source-with-sideinput => simple_source_with_sideinput}/Dockerfile (96%) create mode 100644 pkg/sideinput/examples/simple_source_with_sideinput/Makefile rename pkg/sideinput/examples/{simple-source-with-sideinput/readme.md => simple_source_with_sideinput/README.md} (100%) rename pkg/sideinput/examples/{simple-source-with-sideinput => simple_source_with_sideinput}/go.mod (95%) rename pkg/sideinput/examples/{simple-source-with-sideinput => simple_source_with_sideinput}/go.sum (100%) rename pkg/sideinput/examples/{simple-source-with-sideinput => simple_source_with_sideinput}/impl/simple_source_sideinput.go (100%) rename pkg/sideinput/examples/{simple-source-with-sideinput => simple_source_with_sideinput}/impl/simple_source_sideinput_test.go (100%) rename pkg/sideinput/examples/{simple-source-with-sideinput => simple_source_with_sideinput}/main.go (100%) delete mode 100644 pkg/sideinput/examples/sink-side-input/Makefile rename pkg/{sinker/examples/redis-e2e-test-sink => sideinput/examples/sink_sideinput}/Dockerfile (72%) create mode 100644 pkg/sideinput/examples/sink_sideinput/Makefile rename pkg/sideinput/examples/{sink-side-input => sink_sideinput}/README.md (65%) rename pkg/sideinput/examples/{sink-side-input => sink_sideinput}/go.mod (96%) rename pkg/sideinput/examples/{sink-side-input => sink_sideinput}/go.sum (100%) rename pkg/sideinput/examples/{sink-side-input => sink_sideinput}/main.go (100%) delete mode 100644 pkg/sinker/examples/redis-e2e-test-sink/Makefile rename pkg/{sideinput/examples/sink-side-input => sinker/examples/redis-sink}/Dockerfile (78%) create mode 100644 pkg/sinker/examples/redis-sink/Makefile rename pkg/sinker/examples/{redis-e2e-test-sink => redis-sink}/README.md (100%) rename pkg/sinker/examples/{redis-e2e-test-sink => redis-sink}/go.mod (100%) rename pkg/sinker/examples/{redis-e2e-test-sink => redis-sink}/go.sum (100%) rename pkg/sinker/examples/{redis-e2e-test-sink => redis-sink}/main.go (100%) diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml index 4b9b3ec3..9ada4718 100644 --- a/.github/workflows/build-push.yaml +++ b/.github/workflows/build-push.yaml @@ -21,11 +21,11 @@ jobs: "pkg/mapper/examples/retry", "pkg/mapper/examples/tickgen", "pkg/mapstreamer/examples/flatmap_stream", "pkg/reducer/examples/counter", "pkg/reducer/examples/sum", "pkg/reducestreamer/examples/counter", "pkg/reducestreamer/examples/sum", "pkg/sessionreducer/examples/counter", "pkg/sessionreducer/examples/sum", - "pkg/sideinput/examples/simple-sideinput/udf", "pkg/sideinput/examples/simple-sideinput", "pkg/sinker/examples/log", + "pkg/sideinput/examples/simple_sideinput/udf", "pkg/sideinput/examples/simple_sideinput", "pkg/sinker/examples/log", "pkg/sourcer/examples/simple_source", "pkg/sourcetransformer/examples/assign_event_time", "pkg/sourcetransformer/examples/event_time_filter", - "pkg/sinker/examples/fallback", "pkg/sideinput/examples/map-side-input", "pkg/sideinput/examples/reduce-side-input", - "pkg/sideinput/examples/sideInput-function", "pkg/sideinput/examples/simple-source-with-sideinput", - "pkg/sideinput/examples/sink-side-input", "pkg/sinker/examples/redis-e2e-test-sink" + "pkg/sinker/examples/fallback", "pkg/sideinput/examples/map_sideinput", "pkg/sideinput/examples/reduce_sideinput", + "pkg/sideinput/examples/sideinput_function", "pkg/sideinput/examples/simple_source_with_sideinput", + "pkg/sideinput/examples/sink_sideinput", "pkg/sinker/examples/redis-sink" ] steps: diff --git a/pkg/sideinput/examples/map-side-input/Makefile b/pkg/sideinput/examples/map-side-input/Makefile deleted file mode 100644 index 7293ed5d..00000000 --- a/pkg/sideinput/examples/map-side-input/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -DOCKERIO_ORG=quay.io/numaio -PLATFORM=linux/x86_64 -TARGET=sideinput-example - -IMAGE_TAG=$(TAG) -ifeq ($(IMAGE_TAG),) - IMAGE_TAG=dev -endif - -.PHONY: build -build: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/e2e-map-sideinput-example main.go - -.PHONY: image -#To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -image: build - docker buildx build -t "$(DOCKERIO_ORG)/numaproj-contrib/e2e-map-sideinput-example:$(IMAGE_TAG)" --platform $(PLATFORM) --target $(TARGET) . --load - -.PHONY: lint -lint: - go mod tidy - golangci-lint run --fix --verbose --concurrency 4 --timeout 5m - -clean: - -rm -rf ./dist \ No newline at end of file diff --git a/pkg/sideinput/examples/map-side-input/udf/Makefile b/pkg/sideinput/examples/map-side-input/udf/Makefile deleted file mode 100644 index 35b66d9b..00000000 --- a/pkg/sideinput/examples/map-side-input/udf/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -DOCKERIO_ORG=quay.io/numaio -PLATFORM=linux/x86_64 -TARGET=map-sideinput-udf - -IMAGE_TAG=$(TAG) -ifeq ($(IMAGE_TAG),) - IMAGE_TAG="dev" -endif - -.PHONY: build -build: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/map-sideinput-udf main.go - -.PHONY: image -#To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -image: build - docker buildx build -t "$(DOCKERIO_ORG)/numaproj-contrib/e2e-map-sideinput-udf:$(IMAGE_TAG)" --platform $(PLATFORM) --target $(TARGET) . --load - -.PHONY: lint -lint: - go mod tidy - golangci-lint run --fix --verbose --concurrency 4 --timeout 5m - -clean: - -rm -rf ./dist \ No newline at end of file diff --git a/pkg/sideinput/examples/map_sideinput/Dockerfile b/pkg/sideinput/examples/map_sideinput/Dockerfile new file mode 100644 index 00000000..a2129aba --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/map-sideinput-example /bin/map-sideinput-example +RUN chmod +x /bin/map-sideinput-example + +#################################################################################################### +# sideinput +#################################################################################################### +FROM scratch as sideinput +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/map-sideinput-example /bin/map-sideinput-example +ENTRYPOINT [ "/bin/map-sideinput-example" ] diff --git a/pkg/sideinput/examples/map_sideinput/Makefile b/pkg/sideinput/examples/map_sideinput/Makefile new file mode 100644 index 00000000..56fb208e --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/map-sideinput-example main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/map-sideinput:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/map-sideinput:${TAG}" --target sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-sideinput:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/map-side-input/README.md b/pkg/sideinput/examples/map_sideinput/README.md similarity index 80% rename from pkg/sideinput/examples/map-side-input/README.md rename to pkg/sideinput/examples/map_sideinput/README.md index 413ebb50..139706a8 100644 --- a/pkg/sideinput/examples/map-side-input/README.md +++ b/pkg/sideinput/examples/map_sideinput/README.md @@ -29,8 +29,7 @@ spec: sideInputs: - name: myticker container: - # A map side input example , see https://github.com/numaproj-contrib/e2e-tests-go/tree/main/map-side-input - image: "quay.io/numaio/numaproj-contrib/e2e-map-sideinput-example:v0.0.2" + image: "quay.io/numaio/numaflow-go/map-sideinput:stable" imagePullPolicy: Always trigger: schedule: "@every 5s" @@ -40,8 +39,7 @@ Vertex spec for the UDF vertex: - name: si-e2e udf: container: - # A map side input udf , see https://github.com/numaproj-contrib/e2e-tests-go/tree/main/map-side-input/udf - image: "quay.io/numaio/numaproj-contrib/e2e-map-sideinput-udf:v0.0.2" + image: "quay.io/numaio/numaflow-go/map-sideinput-udf:stable" imagePullPolicy: Always sideInputs: - myticker diff --git a/pkg/sideinput/examples/map-side-input/go.mod b/pkg/sideinput/examples/map_sideinput/go.mod similarity index 95% rename from pkg/sideinput/examples/map-side-input/go.mod rename to pkg/sideinput/examples/map_sideinput/go.mod index 493f550e..f7b020ad 100644 --- a/pkg/sideinput/examples/map-side-input/go.mod +++ b/pkg/sideinput/examples/map_sideinput/go.mod @@ -1,4 +1,4 @@ -module map-side-input +module map_sideinput go 1.21.2 diff --git a/pkg/sideinput/examples/map-side-input/go.sum b/pkg/sideinput/examples/map_sideinput/go.sum similarity index 100% rename from pkg/sideinput/examples/map-side-input/go.sum rename to pkg/sideinput/examples/map_sideinput/go.sum diff --git a/pkg/sideinput/examples/map-side-input/main.go b/pkg/sideinput/examples/map_sideinput/main.go similarity index 100% rename from pkg/sideinput/examples/map-side-input/main.go rename to pkg/sideinput/examples/map_sideinput/main.go diff --git a/pkg/sideinput/examples/map-side-input/udf/Dockerfile b/pkg/sideinput/examples/map_sideinput/udf/Dockerfile similarity index 96% rename from pkg/sideinput/examples/map-side-input/udf/Dockerfile rename to pkg/sideinput/examples/map_sideinput/udf/Dockerfile index 1cf10fcf..108f0b63 100644 --- a/pkg/sideinput/examples/map-side-input/udf/Dockerfile +++ b/pkg/sideinput/examples/map_sideinput/udf/Dockerfile @@ -12,7 +12,7 @@ RUN chmod +x /bin/map-sideinput-udf #################################################################################################### # udf-sideinput #################################################################################################### -FROM scratch as map-sideinput-udf +FROM scratch as sideinput-udf ARG ARCH COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt diff --git a/pkg/sideinput/examples/map_sideinput/udf/Makefile b/pkg/sideinput/examples/map_sideinput/udf/Makefile new file mode 100644 index 00000000..f38019e7 --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/udf/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/map-sideinput-udf main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/map-sideinput-udf:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput-udf . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/map-sideinput-udf:${TAG}" --target sideinput-udf . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-sideinput-udf:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/map-side-input/udf/go.mod b/pkg/sideinput/examples/map_sideinput/udf/go.mod similarity index 88% rename from pkg/sideinput/examples/map-side-input/udf/go.mod rename to pkg/sideinput/examples/map_sideinput/udf/go.mod index b256632d..46e87cb0 100644 --- a/pkg/sideinput/examples/map-side-input/udf/go.mod +++ b/pkg/sideinput/examples/map_sideinput/udf/go.mod @@ -2,6 +2,8 @@ module udf go 1.21.2 +replace github.com/numaproj/numaflow-go => ../../../../.. + require ( github.com/fsnotify/fsnotify v1.6.0 github.com/numaproj/numaflow-go v0.7.0-rc2 diff --git a/pkg/sideinput/examples/map-side-input/udf/go.sum b/pkg/sideinput/examples/map_sideinput/udf/go.sum similarity index 93% rename from pkg/sideinput/examples/map-side-input/udf/go.sum rename to pkg/sideinput/examples/map_sideinput/udf/go.sum index 23204122..dbcfb187 100644 --- a/pkg/sideinput/examples/map-side-input/udf/go.sum +++ b/pkg/sideinput/examples/map_sideinput/udf/go.sum @@ -8,8 +8,6 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/numaproj/numaflow-go v0.7.0-rc2 h1:oLP0uM9ToFEtiSyzX46cFOlr7ctgcH02cY/XFA1qF8E= -github.com/numaproj/numaflow-go v0.7.0-rc2/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= diff --git a/pkg/sideinput/examples/map-side-input/udf/main.go b/pkg/sideinput/examples/map_sideinput/udf/main.go similarity index 100% rename from pkg/sideinput/examples/map-side-input/udf/main.go rename to pkg/sideinput/examples/map_sideinput/udf/main.go diff --git a/pkg/sideinput/examples/reduce-side-input/Dockerfile b/pkg/sideinput/examples/reduce-side-input/Dockerfile deleted file mode 100644 index 9706fcb3..00000000 --- a/pkg/sideinput/examples/reduce-side-input/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -#################################################################################################### -# base -#################################################################################################### -FROM alpine:3.12.3 as base -RUN apk update && apk upgrade && \ - apk add ca-certificates && \ - apk --no-cache add tzdata - -COPY dist/e2e-reduce-sideinput-example /bin/e2e-reduce-sideinput-example -RUN chmod +x /bin/e2e-reduce-sideinput-example - -#################################################################################################### -# sideinput -#################################################################################################### -FROM scratch as sideinput-example -ARG ARCH -COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo -COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY --from=base /bin/e2e-reduce-sideinput-example /bin/e2e-reduce-sideinput-example -ENTRYPOINT [ "/bin/e2e-reduce-sideinput-example" ] diff --git a/pkg/sideinput/examples/reduce-side-input/Makefile b/pkg/sideinput/examples/reduce-side-input/Makefile deleted file mode 100644 index 6a54389b..00000000 --- a/pkg/sideinput/examples/reduce-side-input/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -DOCKERIO_ORG=quay.io/numaio -PLATFORM=linux/x86_64 -TARGET=sideinput-example - -IMAGE_TAG=$(TAG) -ifeq ($(IMAGE_TAG),) - IMAGE_TAG="dev" -endif - -.PHONY: build -build: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/e2e-reduce-sideinput-example main.go - -.PHONY: image -#To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -image: build - docker buildx build -t "$(DOCKERIO_ORG)/numaproj-contrib/e2e-reduce-sideinput-example:$(IMAGE_TAG)" --platform $(PLATFORM) --target $(TARGET) . --load - -.PHONY: lint -lint: - go mod tidy - golangci-lint run --fix --verbose --concurrency 4 --timeout 5m - -clean: - -rm -rf ./dist \ No newline at end of file diff --git a/pkg/sideinput/examples/reduce-side-input/udf/Makefile b/pkg/sideinput/examples/reduce-side-input/udf/Makefile deleted file mode 100644 index caa6af9b..00000000 --- a/pkg/sideinput/examples/reduce-side-input/udf/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -DOCKERIO_ORG=quay.io/numaio -PLATFORM=linux/x86_64 -TARGET=reduce-sideinput-udf - -IMAGE_TAG=$(TAG) -ifeq ($(IMAGE_TAG),) - IMAGE_TAG="dev" -endif - -.PHONY: build -build: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/reduce-sideinput-udf main.go - -.PHONY: image -#To build an image that supports multiple platforms(linux/amd64,linux/arm64) and push to quay.io, use the following command -image: build - docker buildx build -t "$(DOCKERIO_ORG)/numaproj-contrib/e2e-reduce-sideinput-udf:$(IMAGE_TAG)" --platform $(PLATFORM) --target reduce-sideinput-udf . --load - -.PHONY: lint -lint: - go mod tidy - golangci-lint run --fix --verbose --concurrency 4 --timeout 5m - -clean: - -rm -rf ./dist \ No newline at end of file diff --git a/pkg/sideinput/examples/map-side-input/Dockerfile b/pkg/sideinput/examples/reduce_sideinput/Dockerfile similarity index 72% rename from pkg/sideinput/examples/map-side-input/Dockerfile rename to pkg/sideinput/examples/reduce_sideinput/Dockerfile index 9e3a69b3..ca830d0b 100644 --- a/pkg/sideinput/examples/map-side-input/Dockerfile +++ b/pkg/sideinput/examples/reduce_sideinput/Dockerfile @@ -6,15 +6,15 @@ RUN apk update && apk upgrade && \ apk add ca-certificates && \ apk --no-cache add tzdata -COPY dist/e2e-map-sideinput-example /bin/e2e-map-sideinput-example -RUN chmod +x /bin/e2e-map-sideinput-example +COPY dist/reduce-sideinput-example /bin/reduce-sideinput-example +RUN chmod +x /bin/reduce-sideinput-example #################################################################################################### # sideinput #################################################################################################### -FROM scratch as sideinput-example +FROM scratch as sideinput ARG ARCH COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY --from=base /bin/e2e-map-sideinput-example /bin/e2e-map-sideinput-example -ENTRYPOINT [ "/bin/e2e-map-sideinput-example" ] +COPY --from=base /bin/reduce-sideinput-example /bin/reduce-sideinput-example +ENTRYPOINT [ "/bin/reduce-sideinput-example" ] diff --git a/pkg/sideinput/examples/reduce_sideinput/Makefile b/pkg/sideinput/examples/reduce_sideinput/Makefile new file mode 100644 index 00000000..43d85e38 --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/reduce-sideinput-example main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/reduce-sideinput:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/reduce-sideinput:${TAG}" --target sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-sideinput:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/reduce-side-input/README.md b/pkg/sideinput/examples/reduce_sideinput/README.md similarity index 80% rename from pkg/sideinput/examples/reduce-side-input/README.md rename to pkg/sideinput/examples/reduce_sideinput/README.md index 2d5dc8dc..ea433265 100644 --- a/pkg/sideinput/examples/reduce-side-input/README.md +++ b/pkg/sideinput/examples/reduce_sideinput/README.md @@ -25,8 +25,7 @@ spec: sideInputs: - name: myticker container: - # A reduce side input, see https://github.com/numaproj-contrib/e2e-tests-go/tree/main/reduce-side-input - image: "quay.io/numaio/numaproj-contrib/e2e-reduce-sideinput-example:v0.0.2" + image: "quay.io/numaio/numaflow-go/reduce-sideinput:stable" imagePullPolicy: Always trigger: schedule: "@every 5s" @@ -36,8 +35,7 @@ Vertex spec for the UDF vertex: - name: si-e2e udf: container: - # A reduce side input udf, see https://github.com/numaproj-contrib/e2e-tests-go/tree/main/reduce-side-input/udf - image: "quay.io/numaio/numaproj-contrib/e2e-reduce-sideinput-udf:v0.0.2" + image: "quay.io/numaio/numaproj-contrib/reduce-sideinput-udf:stable" imagePullPolicy: Always groupBy: window: diff --git a/pkg/sideinput/examples/reduce-side-input/go.mod b/pkg/sideinput/examples/reduce_sideinput/go.mod similarity index 94% rename from pkg/sideinput/examples/reduce-side-input/go.mod rename to pkg/sideinput/examples/reduce_sideinput/go.mod index 23b2e218..570247ef 100644 --- a/pkg/sideinput/examples/reduce-side-input/go.mod +++ b/pkg/sideinput/examples/reduce_sideinput/go.mod @@ -1,4 +1,4 @@ -module reduce-side-input +module reduce_sideinput go 1.21 diff --git a/pkg/sideinput/examples/reduce-side-input/go.sum b/pkg/sideinput/examples/reduce_sideinput/go.sum similarity index 100% rename from pkg/sideinput/examples/reduce-side-input/go.sum rename to pkg/sideinput/examples/reduce_sideinput/go.sum diff --git a/pkg/sideinput/examples/reduce-side-input/main.go b/pkg/sideinput/examples/reduce_sideinput/main.go similarity index 100% rename from pkg/sideinput/examples/reduce-side-input/main.go rename to pkg/sideinput/examples/reduce_sideinput/main.go diff --git a/pkg/sideinput/examples/reduce-side-input/udf/Dockerfile b/pkg/sideinput/examples/reduce_sideinput/udf/Dockerfile similarity index 92% rename from pkg/sideinput/examples/reduce-side-input/udf/Dockerfile rename to pkg/sideinput/examples/reduce_sideinput/udf/Dockerfile index 342e87e4..7ae826ea 100644 --- a/pkg/sideinput/examples/reduce-side-input/udf/Dockerfile +++ b/pkg/sideinput/examples/reduce_sideinput/udf/Dockerfile @@ -7,12 +7,12 @@ RUN apk update && apk upgrade && \ apk --no-cache add tzdata COPY dist/reduce-sideinput-udf /bin/reduce-sideinput-udf -RUN chmod +x /bin/reduce-sideinput-udf +RUN chmod +x /bin/reduce_sideinput-udf #################################################################################################### # udf-sideinput #################################################################################################### -FROM scratch as reduce-sideinput-udf +FROM scratch as sideinput-udf ARG ARCH COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt diff --git a/pkg/sideinput/examples/reduce_sideinput/udf/Makefile b/pkg/sideinput/examples/reduce_sideinput/udf/Makefile new file mode 100644 index 00000000..12336203 --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/udf/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/reduce-sideinput-udf main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/reduce-sideinput-udf:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput-udf . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/reduce-sideinput-udf:${TAG}" --target sideinput-udf . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-sideinput-udf:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/reduce-side-input/udf/go.mod b/pkg/sideinput/examples/reduce_sideinput/udf/go.mod similarity index 89% rename from pkg/sideinput/examples/reduce-side-input/udf/go.mod rename to pkg/sideinput/examples/reduce_sideinput/udf/go.mod index ced076aa..b6b1d6dd 100644 --- a/pkg/sideinput/examples/reduce-side-input/udf/go.mod +++ b/pkg/sideinput/examples/reduce_sideinput/udf/go.mod @@ -2,6 +2,8 @@ module udf go 1.21.2 +replace github.com/numaproj/numaflow-go => ../../../../.. + require ( github.com/fsnotify/fsnotify v1.7.0 github.com/numaproj/numaflow-go v0.7.0-rc2 diff --git a/pkg/sideinput/examples/reduce-side-input/udf/go.sum b/pkg/sideinput/examples/reduce_sideinput/udf/go.sum similarity index 93% rename from pkg/sideinput/examples/reduce-side-input/udf/go.sum rename to pkg/sideinput/examples/reduce_sideinput/udf/go.sum index 2ff1915f..18817608 100644 --- a/pkg/sideinput/examples/reduce-side-input/udf/go.sum +++ b/pkg/sideinput/examples/reduce_sideinput/udf/go.sum @@ -8,8 +8,6 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/numaproj/numaflow-go v0.7.0-rc2 h1:oLP0uM9ToFEtiSyzX46cFOlr7ctgcH02cY/XFA1qF8E= -github.com/numaproj/numaflow-go v0.7.0-rc2/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= diff --git a/pkg/sideinput/examples/reduce-side-input/udf/main.go b/pkg/sideinput/examples/reduce_sideinput/udf/main.go similarity index 98% rename from pkg/sideinput/examples/reduce-side-input/udf/main.go rename to pkg/sideinput/examples/reduce_sideinput/udf/main.go index 4c6bfee9..0d11b6d2 100644 --- a/pkg/sideinput/examples/reduce-side-input/udf/main.go +++ b/pkg/sideinput/examples/reduce_sideinput/udf/main.go @@ -63,7 +63,7 @@ func (s *Sum) Reduce(ctx context.Context, keys []string, reduceCh <-chan reducer } if s.sum > 0 && len(siData) > 0 { - msg := []byte("reduce-side-input") + msg := []byte("reduce_sideinput") return reducer.MessagesBuilder().Append(reducer.NewMessage(msg)) } else { return reducer.MessagesBuilder().Append(reducer.MessageToDrop()) diff --git a/pkg/sideinput/examples/sideInput-function/Makefile b/pkg/sideinput/examples/sideInput-function/Makefile deleted file mode 100644 index de3a5708..00000000 --- a/pkg/sideinput/examples/sideInput-function/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -DOCKERIO_ORG=quay.io/numaio -PLATFORM=linux/amd64 -TARGET=sideinput - -IMAGE_TAG=$(TAG) -ifeq ($(IMAGE_TAG),) - IMAGE_TAG=latest -endif - -.PHONY: build image lint clean - -build: clean - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sideinput-example main.go - -image: build - docker buildx build --no-cache -t "$(DOCKERIO_ORG)/numaflow-sideinput/sideinput-sink-e2e-test:$(IMAGE_TAG)" --platform $(PLATFORM) --target $(TARGET) . --load - -lint: - go mod tidy - golangci-lint run --fix --verbose --concurrency 4 --timeout 5m - -clean: - -rm -rf ./dist diff --git a/pkg/sideinput/examples/sideInput-function/Dockerfile b/pkg/sideinput/examples/sideinput_function/Dockerfile similarity index 78% rename from pkg/sideinput/examples/sideInput-function/Dockerfile rename to pkg/sideinput/examples/sideinput_function/Dockerfile index 76725341..31c55a31 100644 --- a/pkg/sideinput/examples/sideInput-function/Dockerfile +++ b/pkg/sideinput/examples/sideinput_function/Dockerfile @@ -6,8 +6,8 @@ RUN apk update && apk upgrade && \ apk add ca-certificates && \ apk --no-cache add tzdata -COPY dist/sideinput-example /bin/sideinput-example -RUN chmod +x /bin/sideinput-example +COPY dist/sideinput-function /bin/sideinput-function +RUN chmod +x /bin/sideinput-function #################################################################################################### # sideinput @@ -16,5 +16,5 @@ FROM scratch as sideinput ARG ARCH COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY --from=base /bin/sideinput-example /bin/sideinput-example -ENTRYPOINT [ "/bin/sideinput-example" ] \ No newline at end of file +COPY --from=base /bin/sideinput-function /bin/sideinput-function +ENTRYPOINT [ "/bin/sideinput-function" ] \ No newline at end of file diff --git a/pkg/sideinput/examples/sideinput_function/Makefile b/pkg/sideinput/examples/sideinput_function/Makefile new file mode 100644 index 00000000..c6187577 --- /dev/null +++ b/pkg/sideinput/examples/sideinput_function/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sideinput-function main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/sideinput-function:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/sideinput-function:${TAG}" --target sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/sideinput-function:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/sideInput-function/readme.md b/pkg/sideinput/examples/sideinput_function/README.md similarity index 93% rename from pkg/sideinput/examples/sideInput-function/readme.md rename to pkg/sideinput/examples/sideinput_function/README.md index 5420fd4e..b722e798 100644 --- a/pkg/sideinput/examples/sideInput-function/readme.md +++ b/pkg/sideinput/examples/sideinput_function/README.md @@ -51,7 +51,7 @@ spec: sideInputs: - name: myticker container: - image: "quay.io/numaio/numaflow-go/sideinput:v0.5.0" + image: "quay.io/numaio/numaflow-go/sideinput-function:stable" imagePullPolicy: Always trigger: schedule: "*/2 * * * *" @@ -63,7 +63,7 @@ Vertex spec for the UDF vertex: - name: si-log udf: container: - image: "quay.io/numaio/numaflow-go/udf-sideinput:v0.5.0" + image: "quay.io/numaio/numaflow-go/sideinput-function-udf:stable" imagePullPolicy: Always containerTemplate: env: diff --git a/pkg/sideinput/examples/sideInput-function/go.mod b/pkg/sideinput/examples/sideinput_function/go.mod similarity index 94% rename from pkg/sideinput/examples/sideInput-function/go.mod rename to pkg/sideinput/examples/sideinput_function/go.mod index 71f547da..d2b882d8 100644 --- a/pkg/sideinput/examples/sideInput-function/go.mod +++ b/pkg/sideinput/examples/sideinput_function/go.mod @@ -1,4 +1,4 @@ -module sideinput-function +module even_odd_sideinput go 1.21 diff --git a/pkg/sideinput/examples/sideInput-function/go.sum b/pkg/sideinput/examples/sideinput_function/go.sum similarity index 100% rename from pkg/sideinput/examples/sideInput-function/go.sum rename to pkg/sideinput/examples/sideinput_function/go.sum diff --git a/pkg/sideinput/examples/sideInput-function/main.go b/pkg/sideinput/examples/sideinput_function/main.go similarity index 100% rename from pkg/sideinput/examples/sideInput-function/main.go rename to pkg/sideinput/examples/sideinput_function/main.go diff --git a/pkg/sideinput/examples/simple-source-with-sideinput/Makefile b/pkg/sideinput/examples/simple-source-with-sideinput/Makefile deleted file mode 100644 index a52489b8..00000000 --- a/pkg/sideinput/examples/simple-source-with-sideinput/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -DOCKERIO_ORG=quay.io/numaio -PLATFORM=linux/amd64 -TARGET=simple-source - -IMAGE_TAG=$(TAG) -ifeq ($(IMAGE_TAG),) - IMAGE_TAG=latest -endif - -.PHONY: build image lint clean - -build: clean - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/simple-source-example main.go - -image: build - docker buildx build --no-cache -t "$(DOCKERIO_ORG)/numaflow-source/simple_source_with_sideinput:$(IMAGE_TAG)" --platform $(PLATFORM) --target $(TARGET) . --load - -lint: - go mod tidy - golangci-lint run --fix --verbose --concurrency 4 --timeout 5m - -clean: - -rm -rf ./dist diff --git a/pkg/sideinput/examples/simple-sideinput/Dockerfile b/pkg/sideinput/examples/simple_sideinput/Dockerfile similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/Dockerfile rename to pkg/sideinput/examples/simple_sideinput/Dockerfile diff --git a/pkg/sideinput/examples/simple-sideinput/Makefile b/pkg/sideinput/examples/simple_sideinput/Makefile similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/Makefile rename to pkg/sideinput/examples/simple_sideinput/Makefile diff --git a/pkg/sideinput/examples/simple-sideinput/README.md b/pkg/sideinput/examples/simple_sideinput/README.md similarity index 94% rename from pkg/sideinput/examples/simple-sideinput/README.md rename to pkg/sideinput/examples/simple_sideinput/README.md index c80afb5d..93029dbf 100644 --- a/pkg/sideinput/examples/simple-sideinput/README.md +++ b/pkg/sideinput/examples/simple_sideinput/README.md @@ -49,7 +49,7 @@ spec: sideInputs: - name: myticker container: - image: "quay.io/numaio/numaflow-go/sideinput:v0.5.0" + image: "quay.io/numaio/numaflow-go/sideinput-example:v0.5.0" imagePullPolicy: Always trigger: schedule: "*/2 * * * *" @@ -61,7 +61,7 @@ Vertex spec for the UDF vertex: - name: si-log udf: container: - image: "quay.io/numaio/numaflow-go/udf-sideinput:v0.5.0" + image: "quay.io/numaio/numaflow-go/sideinput-example-udf:v0.5.0" imagePullPolicy: Always containerTemplate: env: diff --git a/pkg/sideinput/examples/simple-sideinput/go.mod b/pkg/sideinput/examples/simple_sideinput/go.mod similarity index 95% rename from pkg/sideinput/examples/simple-sideinput/go.mod rename to pkg/sideinput/examples/simple_sideinput/go.mod index c1aa3740..2194a0b1 100644 --- a/pkg/sideinput/examples/simple-sideinput/go.mod +++ b/pkg/sideinput/examples/simple_sideinput/go.mod @@ -1,4 +1,4 @@ -module simple-sideinput +module simple_sideinput go 1.20 diff --git a/pkg/sideinput/examples/simple-sideinput/go.sum b/pkg/sideinput/examples/simple_sideinput/go.sum similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/go.sum rename to pkg/sideinput/examples/simple_sideinput/go.sum diff --git a/pkg/sideinput/examples/simple-sideinput/main.go b/pkg/sideinput/examples/simple_sideinput/main.go similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/main.go rename to pkg/sideinput/examples/simple_sideinput/main.go diff --git a/pkg/sideinput/examples/simple-sideinput/pipeline.yaml b/pkg/sideinput/examples/simple_sideinput/pipeline.yaml similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/pipeline.yaml rename to pkg/sideinput/examples/simple_sideinput/pipeline.yaml diff --git a/pkg/sideinput/examples/simple-sideinput/udf/Dockerfile b/pkg/sideinput/examples/simple_sideinput/udf/Dockerfile similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/udf/Dockerfile rename to pkg/sideinput/examples/simple_sideinput/udf/Dockerfile diff --git a/pkg/sideinput/examples/simple-sideinput/udf/Makefile b/pkg/sideinput/examples/simple_sideinput/udf/Makefile similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/udf/Makefile rename to pkg/sideinput/examples/simple_sideinput/udf/Makefile diff --git a/pkg/sideinput/examples/simple-sideinput/udf/go.mod b/pkg/sideinput/examples/simple_sideinput/udf/go.mod similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/udf/go.mod rename to pkg/sideinput/examples/simple_sideinput/udf/go.mod diff --git a/pkg/sideinput/examples/simple-sideinput/udf/go.sum b/pkg/sideinput/examples/simple_sideinput/udf/go.sum similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/udf/go.sum rename to pkg/sideinput/examples/simple_sideinput/udf/go.sum diff --git a/pkg/sideinput/examples/simple-sideinput/udf/main.go b/pkg/sideinput/examples/simple_sideinput/udf/main.go similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/udf/main.go rename to pkg/sideinput/examples/simple_sideinput/udf/main.go diff --git a/pkg/sideinput/examples/simple-source-with-sideinput/Dockerfile b/pkg/sideinput/examples/simple_source_with_sideinput/Dockerfile similarity index 96% rename from pkg/sideinput/examples/simple-source-with-sideinput/Dockerfile rename to pkg/sideinput/examples/simple_source_with_sideinput/Dockerfile index 166d7411..ba4e9c66 100644 --- a/pkg/sideinput/examples/simple-source-with-sideinput/Dockerfile +++ b/pkg/sideinput/examples/simple_source_with_sideinput/Dockerfile @@ -12,7 +12,7 @@ RUN chmod +x /bin/simple-source-example #################################################################################################### # simple-source #################################################################################################### -FROM scratch as simple-source +FROM scratch as sideinput ARG ARCH COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt diff --git a/pkg/sideinput/examples/simple_source_with_sideinput/Makefile b/pkg/sideinput/examples/simple_source_with_sideinput/Makefile new file mode 100644 index 00000000..79c631e6 --- /dev/null +++ b/pkg/sideinput/examples/simple_source_with_sideinput/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/simple-source-example main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/simple-source-with-sideinput:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/simple-source-with-sideinput:${TAG}" --target sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/simple-source-with-sideinput:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/simple-source-with-sideinput/readme.md b/pkg/sideinput/examples/simple_source_with_sideinput/README.md similarity index 100% rename from pkg/sideinput/examples/simple-source-with-sideinput/readme.md rename to pkg/sideinput/examples/simple_source_with_sideinput/README.md diff --git a/pkg/sideinput/examples/simple-source-with-sideinput/go.mod b/pkg/sideinput/examples/simple_source_with_sideinput/go.mod similarity index 95% rename from pkg/sideinput/examples/simple-source-with-sideinput/go.mod rename to pkg/sideinput/examples/simple_source_with_sideinput/go.mod index 5620a833..8f7bbb07 100644 --- a/pkg/sideinput/examples/simple-source-with-sideinput/go.mod +++ b/pkg/sideinput/examples/simple_source_with_sideinput/go.mod @@ -1,4 +1,4 @@ -module simple-source-with-sideinput +module simple_source_with_sideinput go 1.21 diff --git a/pkg/sideinput/examples/simple-source-with-sideinput/go.sum b/pkg/sideinput/examples/simple_source_with_sideinput/go.sum similarity index 100% rename from pkg/sideinput/examples/simple-source-with-sideinput/go.sum rename to pkg/sideinput/examples/simple_source_with_sideinput/go.sum diff --git a/pkg/sideinput/examples/simple-source-with-sideinput/impl/simple_source_sideinput.go b/pkg/sideinput/examples/simple_source_with_sideinput/impl/simple_source_sideinput.go similarity index 100% rename from pkg/sideinput/examples/simple-source-with-sideinput/impl/simple_source_sideinput.go rename to pkg/sideinput/examples/simple_source_with_sideinput/impl/simple_source_sideinput.go diff --git a/pkg/sideinput/examples/simple-source-with-sideinput/impl/simple_source_sideinput_test.go b/pkg/sideinput/examples/simple_source_with_sideinput/impl/simple_source_sideinput_test.go similarity index 100% rename from pkg/sideinput/examples/simple-source-with-sideinput/impl/simple_source_sideinput_test.go rename to pkg/sideinput/examples/simple_source_with_sideinput/impl/simple_source_sideinput_test.go diff --git a/pkg/sideinput/examples/simple-source-with-sideinput/main.go b/pkg/sideinput/examples/simple_source_with_sideinput/main.go similarity index 100% rename from pkg/sideinput/examples/simple-source-with-sideinput/main.go rename to pkg/sideinput/examples/simple_source_with_sideinput/main.go diff --git a/pkg/sideinput/examples/sink-side-input/Makefile b/pkg/sideinput/examples/sink-side-input/Makefile deleted file mode 100644 index 45697e3d..00000000 --- a/pkg/sideinput/examples/sink-side-input/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -DOCKERIO_ORG=quay.io/numaio -PLATFORM=linux/amd64 -TARGET=redis - -IMAGE_TAG=$(TAG) -ifeq ($(IMAGE_TAG),) - IMAGE_TAG=latest -endif - -.PHONY: build image lint clean - -build: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/redis-e2e-test-sink main.go - -image: build - docker buildx build --no-cache -t "$(DOCKERIO_ORG)/numaflow-sink/redis-e2e-test-sink-with-sideinput:$(IMAGE_TAG)" --platform $(PLATFORM) --target $(TARGET) . --load - -lint: - go mod tidy - golangci-lint run --fix --verbose --concurrency 4 --timeout 5m - -clean: - -rm -rf ./dist diff --git a/pkg/sinker/examples/redis-e2e-test-sink/Dockerfile b/pkg/sideinput/examples/sink_sideinput/Dockerfile similarity index 72% rename from pkg/sinker/examples/redis-e2e-test-sink/Dockerfile rename to pkg/sideinput/examples/sink_sideinput/Dockerfile index 7efeff57..e67b592b 100644 --- a/pkg/sinker/examples/redis-e2e-test-sink/Dockerfile +++ b/pkg/sideinput/examples/sink_sideinput/Dockerfile @@ -6,15 +6,15 @@ RUN apk update && apk upgrade && \ apk add ca-certificates && \ apk --no-cache add tzdata -COPY dist/redis-e2e-test-sink /bin/redis-e2e-test-sink -RUN chmod +x /bin/redis-e2e-test-sink +COPY dist/redis-sink-with-sideinput /bin/redis-sink-with-sideinput +RUN chmod +x /bin/redis-sink-with-sideinput #################################################################################################### # redis #################################################################################################### -FROM scratch as redis +FROM scratch as sideinput ARG ARCH COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY --from=base /bin/redis-e2e-test-sink /bin/redis-e2e-test-sink -ENTRYPOINT [ "/bin/redis-e2e-test-sink" ] +COPY --from=base /bin/redis-sink-with-sideinput /bin/redis-sink-with-sideinput +ENTRYPOINT [ "/bin/redis-sink-with-sideinput" ] diff --git a/pkg/sideinput/examples/sink_sideinput/Makefile b/pkg/sideinput/examples/sink_sideinput/Makefile new file mode 100644 index 00000000..9bc0c3d5 --- /dev/null +++ b/pkg/sideinput/examples/sink_sideinput/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/redis-sink-with-sideinput main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/redis-sink-with-sideinput:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/redis-sink-with-sideinput:${TAG}" --target sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/redis-sink-with-sideinput:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/sink-side-input/README.md b/pkg/sideinput/examples/sink_sideinput/README.md similarity index 65% rename from pkg/sideinput/examples/sink-side-input/README.md rename to pkg/sideinput/examples/sink_sideinput/README.md index 44b10fce..c6a2e810 100644 --- a/pkg/sideinput/examples/sink-side-input/README.md +++ b/pkg/sideinput/examples/sink_sideinput/README.md @@ -4,15 +4,14 @@ This Redis UDSink was specifically created for Numaflow end-to-end tests. It interacts with a Redis instance and, upon receiving data, it sets the content of the side input as the key in Redis, and the number of occurrences of that key as its value. - The name of a hash is pipelineName:sinkName. # How It Works: Uses the fsnotify package to watch for changes to the side input file. Reads and stores the content of the side input file when created. For each received message, the sink: -1-Uses the content of the side input as the Redis hash field (key). -2-Increments the value associated with that key (number of occurrences of that key). -3- The hash name in Redis is determined by the environment variables NUMAFLOW_PIPELINE_NAME and NUMAFLOW_VERTEX_NAME. +1. Uses the content of the side input as the Redis hash field (key) +2. Increments the value associated with that key (number of occurrences of that key) +3. The hash name in Redis is determined by the environment variables `NUMAFLOW_PIPELINE_NAME` and `NUMAFLOW_VERTEX_NAME` diff --git a/pkg/sideinput/examples/sink-side-input/go.mod b/pkg/sideinput/examples/sink_sideinput/go.mod similarity index 96% rename from pkg/sideinput/examples/sink-side-input/go.mod rename to pkg/sideinput/examples/sink_sideinput/go.mod index a0b671a0..6521c8d7 100644 --- a/pkg/sideinput/examples/sink-side-input/go.mod +++ b/pkg/sideinput/examples/sink_sideinput/go.mod @@ -1,4 +1,4 @@ -module sink-side-input +module sink_sideinput go 1.21 diff --git a/pkg/sideinput/examples/sink-side-input/go.sum b/pkg/sideinput/examples/sink_sideinput/go.sum similarity index 100% rename from pkg/sideinput/examples/sink-side-input/go.sum rename to pkg/sideinput/examples/sink_sideinput/go.sum diff --git a/pkg/sideinput/examples/sink-side-input/main.go b/pkg/sideinput/examples/sink_sideinput/main.go similarity index 100% rename from pkg/sideinput/examples/sink-side-input/main.go rename to pkg/sideinput/examples/sink_sideinput/main.go diff --git a/pkg/sinker/examples/redis-e2e-test-sink/Makefile b/pkg/sinker/examples/redis-e2e-test-sink/Makefile deleted file mode 100644 index 89dec74a..00000000 --- a/pkg/sinker/examples/redis-e2e-test-sink/Makefile +++ /dev/null @@ -1,10 +0,0 @@ -.PHONY: build -build: - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/redis-e2e-test-sink main.go - -.PHONY: image -image: build - docker build -t "quay.io/numaio/numaflow-sink/redis-e2e-test-sink:latest" --target redis . - -clean: - -rm -rf ./dist \ No newline at end of file diff --git a/pkg/sideinput/examples/sink-side-input/Dockerfile b/pkg/sinker/examples/redis-sink/Dockerfile similarity index 78% rename from pkg/sideinput/examples/sink-side-input/Dockerfile rename to pkg/sinker/examples/redis-sink/Dockerfile index 7efeff57..3f6ffd4b 100644 --- a/pkg/sideinput/examples/sink-side-input/Dockerfile +++ b/pkg/sinker/examples/redis-sink/Dockerfile @@ -6,8 +6,8 @@ RUN apk update && apk upgrade && \ apk add ca-certificates && \ apk --no-cache add tzdata -COPY dist/redis-e2e-test-sink /bin/redis-e2e-test-sink -RUN chmod +x /bin/redis-e2e-test-sink +COPY dist/redis-sink /bin/redis-sink +RUN chmod +x /bin/redis-sink #################################################################################################### # redis @@ -16,5 +16,5 @@ FROM scratch as redis ARG ARCH COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt -COPY --from=base /bin/redis-e2e-test-sink /bin/redis-e2e-test-sink -ENTRYPOINT [ "/bin/redis-e2e-test-sink" ] +COPY --from=base /bin/redis-sink /bin/redis-sink +ENTRYPOINT [ "/bin/redis-sink" ] diff --git a/pkg/sinker/examples/redis-sink/Makefile b/pkg/sinker/examples/redis-sink/Makefile new file mode 100644 index 00000000..1d26a234 --- /dev/null +++ b/pkg/sinker/examples/redis-sink/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/redis-sink main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/redis-sink:${TAG}" --platform linux/amd64,linux/arm64 --target redis . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/redis-sink:${TAG}" --target redis . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/redis-sink:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sinker/examples/redis-e2e-test-sink/README.md b/pkg/sinker/examples/redis-sink/README.md similarity index 100% rename from pkg/sinker/examples/redis-e2e-test-sink/README.md rename to pkg/sinker/examples/redis-sink/README.md diff --git a/pkg/sinker/examples/redis-e2e-test-sink/go.mod b/pkg/sinker/examples/redis-sink/go.mod similarity index 100% rename from pkg/sinker/examples/redis-e2e-test-sink/go.mod rename to pkg/sinker/examples/redis-sink/go.mod diff --git a/pkg/sinker/examples/redis-e2e-test-sink/go.sum b/pkg/sinker/examples/redis-sink/go.sum similarity index 100% rename from pkg/sinker/examples/redis-e2e-test-sink/go.sum rename to pkg/sinker/examples/redis-sink/go.sum diff --git a/pkg/sinker/examples/redis-e2e-test-sink/main.go b/pkg/sinker/examples/redis-sink/main.go similarity index 100% rename from pkg/sinker/examples/redis-e2e-test-sink/main.go rename to pkg/sinker/examples/redis-sink/main.go