modify makefiles and dockerfiles

Signed-off-by: a3hadi <[email protected]>
numaproj · Apr 23, 2024 · 1b380fc · 1b380fc
1 parent 185dc5b
commit 1b380fc
Show file tree

Hide file tree

Showing 70 changed files with 212 additions and 252 deletions.
diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml
@@ -21,11 +21,11 @@ jobs:
           "pkg/mapper/examples/retry", "pkg/mapper/examples/tickgen", "pkg/mapstreamer/examples/flatmap_stream",
           "pkg/reducer/examples/counter", "pkg/reducer/examples/sum", "pkg/reducestreamer/examples/counter",
           "pkg/reducestreamer/examples/sum", "pkg/sessionreducer/examples/counter", "pkg/sessionreducer/examples/sum",
-          "pkg/sideinput/examples/simple-sideinput/udf", "pkg/sideinput/examples/simple-sideinput", "pkg/sinker/examples/log",
+          "pkg/sideinput/examples/simple_sideinput/udf", "pkg/sideinput/examples/simple_sideinput", "pkg/sinker/examples/log",
           "pkg/sourcer/examples/simple_source", "pkg/sourcetransformer/examples/assign_event_time", "pkg/sourcetransformer/examples/event_time_filter",
-          "pkg/sinker/examples/fallback", "pkg/sideinput/examples/map-side-input", "pkg/sideinput/examples/reduce-side-input",
-          "pkg/sideinput/examples/sideInput-function", "pkg/sideinput/examples/simple-source-with-sideinput",
-          "pkg/sideinput/examples/sink-side-input", "pkg/sinker/examples/redis-e2e-test-sink"
+          "pkg/sinker/examples/fallback", "pkg/sideinput/examples/map_sideinput", "pkg/sideinput/examples/reduce_sideinput",
+          "pkg/sideinput/examples/sideinput_function", "pkg/sideinput/examples/simple_source_with_sideinput",
+          "pkg/sideinput/examples/sink_sideinput", "pkg/sinker/examples/redis-sink"
         ]
 
     steps:

diff --git a/pkg/sideinput/examples/map-side-input/Makefile b/pkg/sideinput/examples/map-side-input/Makefile
diff --git a/pkg/sideinput/examples/map-side-input/udf/Makefile b/pkg/sideinput/examples/map-side-input/udf/Makefile
diff --git a/pkg/sideinput/examples/map_sideinput/Dockerfile b/pkg/sideinput/examples/map_sideinput/Dockerfile
@@ -0,0 +1,20 @@
+####################################################################################################
+# base
+####################################################################################################
+FROM alpine:3.12.3 as base
+RUN apk update && apk upgrade && \
+    apk add ca-certificates && \
+    apk --no-cache add tzdata
+
+COPY dist/map-sideinput-example /bin/map-sideinput-example
+RUN chmod +x /bin/map-sideinput-example
+
+####################################################################################################
+# sideinput
+####################################################################################################
+FROM scratch as sideinput
+ARG ARCH
+COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
+COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=base /bin/map-sideinput-example /bin/map-sideinput-example
+ENTRYPOINT [ "/bin/map-sideinput-example" ]
diff --git a/pkg/sideinput/examples/map_sideinput/Makefile b/pkg/sideinput/examples/map_sideinput/Makefile
@@ -0,0 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
+.PHONY: build
+build:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/map-sideinput-example main.go
+
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/map-sideinput:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push
+
+.PHONY: image
+image: build
+	docker build -t "quay.io/numaio/numaflow-go/map-sideinput:${TAG}" --target sideinput .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-sideinput:${TAG}"; fi
+
+clean:
+	-rm -rf ./dist
diff --git a/...deinput/examples/map-side-input/README.md → ...ideinput/examples/map_sideinput/README.md b/...deinput/examples/map-side-input/README.md → ...ideinput/examples/map_sideinput/README.md
@@ -29,8 +29,7 @@ spec:
   sideInputs:
     - name: myticker
       container:
-        # A map side input example , see https://github.com/numaproj-contrib/e2e-tests-go/tree/main/map-side-input
-        image: "quay.io/numaio/numaproj-contrib/e2e-map-sideinput-example:v0.0.2"
+        image: "quay.io/numaio/numaflow-go/map-sideinput:stable"
         imagePullPolicy: Always
       trigger:
         schedule: "@every 5s"
@@ -40,8 +39,7 @@ Vertex spec for the UDF vertex:
     - name: si-e2e
       udf:
         container:
-          # A map side input udf , see https://github.com/numaproj-contrib/e2e-tests-go/tree/main/map-side-input/udf
-          image: "quay.io/numaio/numaproj-contrib/e2e-map-sideinput-udf:v0.0.2"
+          image: "quay.io/numaio/numaflow-go/map-sideinput-udf:stable"
           imagePullPolicy: Always
       sideInputs:
         - myticker

diff --git a/pkg/sideinput/examples/map-side-input/go.mod → pkg/sideinput/examples/map_sideinput/go.mod b/pkg/sideinput/examples/map-side-input/go.mod → pkg/sideinput/examples/map_sideinput/go.mod
@@ -1,4 +1,4 @@
-module map-side-input
+module map_sideinput
 
 go 1.21.2
 

diff --git a/pkg/sideinput/examples/map-side-input/go.sum → pkg/sideinput/examples/map_sideinput/go.sum b/pkg/sideinput/examples/map-side-input/go.sum → pkg/sideinput/examples/map_sideinput/go.sum
diff --git a/...sideinput/examples/map-side-input/main.go → pkg/sideinput/examples/map_sideinput/main.go b/...sideinput/examples/map-side-input/main.go → pkg/sideinput/examples/map_sideinput/main.go
diff --git a/...ut/examples/map-side-input/udf/Dockerfile → ...put/examples/map_sideinput/udf/Dockerfile b/...ut/examples/map-side-input/udf/Dockerfile → ...put/examples/map_sideinput/udf/Dockerfile
@@ -12,7 +12,7 @@ RUN chmod +x /bin/map-sideinput-udf
 ####################################################################################################
 # udf-sideinput
 ####################################################################################################
-FROM scratch as map-sideinput-udf
+FROM scratch as sideinput-udf
 ARG ARCH
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

diff --git a/pkg/sideinput/examples/map_sideinput/udf/Makefile b/pkg/sideinput/examples/map_sideinput/udf/Makefile
@@ -0,0 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
+.PHONY: build
+build:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/map-sideinput-udf main.go
+
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/map-sideinput-udf:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput-udf . --push
+
+.PHONY: image
+image: build
+	docker build -t "quay.io/numaio/numaflow-go/map-sideinput-udf:${TAG}" --target sideinput-udf .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-sideinput-udf:${TAG}"; fi
+
+clean:
+	-rm -rf ./dist
diff --git a/...einput/examples/map-side-input/udf/go.mod → ...deinput/examples/map_sideinput/udf/go.mod b/...einput/examples/map-side-input/udf/go.mod → ...deinput/examples/map_sideinput/udf/go.mod
@@ -2,6 +2,8 @@ module udf
 
 go 1.21.2
 
+replace github.com/numaproj/numaflow-go => ../../../../..
+
 require (
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/numaproj/numaflow-go v0.7.0-rc2

diff --git a/...einput/examples/map-side-input/udf/go.sum → ...deinput/examples/map_sideinput/udf/go.sum b/...einput/examples/map-side-input/udf/go.sum → ...deinput/examples/map_sideinput/udf/go.sum
@@ -8,8 +8,6 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/numaproj/numaflow-go v0.7.0-rc2 h1:oLP0uM9ToFEtiSyzX46cFOlr7ctgcH02cY/XFA1qF8E=
-github.com/numaproj/numaflow-go v0.7.0-rc2/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=

diff --git a/...input/examples/map-side-input/udf/main.go → ...einput/examples/map_sideinput/udf/main.go b/...input/examples/map-side-input/udf/main.go → ...einput/examples/map_sideinput/udf/main.go
diff --git a/pkg/sideinput/examples/reduce-side-input/Dockerfile b/pkg/sideinput/examples/reduce-side-input/Dockerfile
diff --git a/pkg/sideinput/examples/reduce-side-input/Makefile b/pkg/sideinput/examples/reduce-side-input/Makefile
diff --git a/pkg/sideinput/examples/reduce-side-input/udf/Makefile b/pkg/sideinput/examples/reduce-side-input/udf/Makefile
diff --git a/...einput/examples/map-side-input/Dockerfile → ...nput/examples/reduce_sideinput/Dockerfile b/...einput/examples/map-side-input/Dockerfile → ...nput/examples/reduce_sideinput/Dockerfile
@@ -6,15 +6,15 @@ RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata
 
-COPY dist/e2e-map-sideinput-example /bin/e2e-map-sideinput-example
-RUN chmod +x /bin/e2e-map-sideinput-example
+COPY dist/reduce-sideinput-example /bin/reduce-sideinput-example
+RUN chmod +x /bin/reduce-sideinput-example
 
 ####################################################################################################
 # sideinput
 ####################################################################################################
-FROM scratch as sideinput-example
+FROM scratch as sideinput
 ARG ARCH
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
-COPY --from=base /bin/e2e-map-sideinput-example /bin/e2e-map-sideinput-example
-ENTRYPOINT [ "/bin/e2e-map-sideinput-example" ]
+COPY --from=base /bin/reduce-sideinput-example /bin/reduce-sideinput-example
+ENTRYPOINT [ "/bin/reduce-sideinput-example" ]
diff --git a/pkg/sideinput/examples/reduce_sideinput/Makefile b/pkg/sideinput/examples/reduce_sideinput/Makefile
@@ -0,0 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
+.PHONY: build
+build:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/reduce-sideinput-example main.go
+
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/reduce-sideinput:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push
+
+.PHONY: image
+image: build
+	docker build -t "quay.io/numaio/numaflow-go/reduce-sideinput:${TAG}" --target sideinput .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-sideinput:${TAG}"; fi
+
+clean:
+	-rm -rf ./dist
diff --git a/...nput/examples/reduce-side-input/README.md → ...input/examples/reduce_sideinput/README.md b/...nput/examples/reduce-side-input/README.md → ...input/examples/reduce_sideinput/README.md
@@ -25,8 +25,7 @@ spec:
   sideInputs:
     - name: myticker
       container:
-        # A reduce side input, see https://github.com/numaproj-contrib/e2e-tests-go/tree/main/reduce-side-input
-        image: "quay.io/numaio/numaproj-contrib/e2e-reduce-sideinput-example:v0.0.2"
+        image: "quay.io/numaio/numaflow-go/reduce-sideinput:stable"
         imagePullPolicy: Always
       trigger:
         schedule: "@every 5s"
@@ -36,8 +35,7 @@ Vertex spec for the UDF vertex:
     - name: si-e2e
       udf:
         container:
-          # A reduce side input udf, see https://github.com/numaproj-contrib/e2e-tests-go/tree/main/reduce-side-input/udf
-          image: "quay.io/numaio/numaproj-contrib/e2e-reduce-sideinput-udf:v0.0.2"
+          image: "quay.io/numaio/numaproj-contrib/reduce-sideinput-udf:stable"
           imagePullPolicy: Always
         groupBy:
           window:

diff --git a/...deinput/examples/reduce-side-input/go.mod → ...ideinput/examples/reduce_sideinput/go.mod b/...deinput/examples/reduce-side-input/go.mod → ...ideinput/examples/reduce_sideinput/go.mod
@@ -1,4 +1,4 @@
-module reduce-side-input
+module reduce_sideinput
 
 go 1.21
 

diff --git a/...deinput/examples/reduce-side-input/go.sum → ...ideinput/examples/reduce_sideinput/go.sum b/...deinput/examples/reduce-side-input/go.sum → ...ideinput/examples/reduce_sideinput/go.sum
diff --git a/...einput/examples/reduce-side-input/main.go → ...deinput/examples/reduce_sideinput/main.go b/...einput/examples/reduce-side-input/main.go → ...deinput/examples/reduce_sideinput/main.go
diff --git a/...examples/reduce-side-input/udf/Dockerfile → .../examples/reduce_sideinput/udf/Dockerfile b/...examples/reduce-side-input/udf/Dockerfile → .../examples/reduce_sideinput/udf/Dockerfile
@@ -7,12 +7,12 @@ RUN apk update && apk upgrade && \
     apk --no-cache add tzdata
 
 COPY dist/reduce-sideinput-udf /bin/reduce-sideinput-udf
-RUN chmod +x /bin/reduce-sideinput-udf
+RUN chmod +x /bin/reduce_sideinput-udf
 
 ####################################################################################################
 # udf-sideinput
 ####################################################################################################
-FROM scratch as reduce-sideinput-udf
+FROM scratch as sideinput-udf
 ARG ARCH
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt

diff --git a/pkg/sideinput/examples/reduce_sideinput/udf/Makefile b/pkg/sideinput/examples/reduce_sideinput/udf/Makefile
@@ -0,0 +1,18 @@
+TAG ?= stable
+PUSH ?= false
+
+.PHONY: build
+build:
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/reduce-sideinput-udf main.go
+
+.PHONY: image-push
+image-push: build
+	docker buildx build -t "quay.io/numaio/numaflow-go/reduce-sideinput-udf:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput-udf . --push
+
+.PHONY: image
+image: build
+	docker build -t "quay.io/numaio/numaflow-go/reduce-sideinput-udf:${TAG}" --target sideinput-udf .
+	@if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-sideinput-udf:${TAG}"; fi
+
+clean:
+	-rm -rf ./dist
diff --git a/...put/examples/reduce-side-input/udf/go.mod → ...nput/examples/reduce_sideinput/udf/go.mod b/...put/examples/reduce-side-input/udf/go.mod → ...nput/examples/reduce_sideinput/udf/go.mod
@@ -2,6 +2,8 @@ module udf
 
 go 1.21.2
 
+replace github.com/numaproj/numaflow-go => ../../../../..
+
 require (
 	github.com/fsnotify/fsnotify v1.7.0
 	github.com/numaproj/numaflow-go v0.7.0-rc2

diff --git a/...put/examples/reduce-side-input/udf/go.sum → ...nput/examples/reduce_sideinput/udf/go.sum b/...put/examples/reduce-side-input/udf/go.sum → ...nput/examples/reduce_sideinput/udf/go.sum
@@ -8,8 +8,6 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/numaproj/numaflow-go v0.7.0-rc2 h1:oLP0uM9ToFEtiSyzX46cFOlr7ctgcH02cY/XFA1qF8E=
-github.com/numaproj/numaflow-go v0.7.0-rc2/go.mod h1:WoMt31+h3up202zTRI8c/qe42B8UbvwLe2mJH0MAlhI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=

diff --git a/...ut/examples/reduce-side-input/udf/main.go → ...put/examples/reduce_sideinput/udf/main.go b/...ut/examples/reduce-side-input/udf/main.go → ...put/examples/reduce_sideinput/udf/main.go
@@ -63,7 +63,7 @@ func (s *Sum) Reduce(ctx context.Context, keys []string, reduceCh <-chan reducer
 	}
 
 	if s.sum > 0 && len(siData) > 0 {
-		msg := []byte("reduce-side-input")
+		msg := []byte("reduce_sideinput")
 		return reducer.MessagesBuilder().Append(reducer.NewMessage(msg))
 	} else {
 		return reducer.MessagesBuilder().Append(reducer.MessageToDrop())