fix: multi-architecture builds (#142)

numaproj · Aug 20, 2024 · b559600 · b559600
1 parent d41d11a
commit b559600
Show file tree

Hide file tree

Showing 66 changed files with 227 additions and 143 deletions.
diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml
@@ -24,8 +24,8 @@ jobs:
           "pkg/sourcer/examples/simple_source", "pkg/sourcetransformer/examples/assign_event_time", "pkg/sourcetransformer/examples/event_time_filter",
           "pkg/sinker/examples/fallback", "pkg/sideinput/examples/map_sideinput", "pkg/sideinput/examples/reduce_sideinput",
           "pkg/sideinput/examples/sideinput_function", "pkg/sideinput/examples/simple_source_with_sideinput",
-          "pkg/sideinput/examples/sink_sideinput", "pkg/sinker/examples/redis-sink", "pkg/sideinput/examples/map_sideinput/udf",
-          "pkg/sideinput/examples/reduce_sideinput/udf", "pkg/batchmapper/examples/batchmap-flatmap"
+          "pkg/sideinput/examples/sink_sideinput", "pkg/sinker/examples/redis_sink", "pkg/sideinput/examples/map_sideinput/udf",
+          "pkg/sideinput/examples/reduce_sideinput/udf", "pkg/batchmapper/examples/batchmap_flatmap"
         ]
 
     steps:

diff --git a/...pper/examples/batchmap-flatmap/Dockerfile → ...pper/examples/batchmap_flatmap/Dockerfile b/...pper/examples/batchmap-flatmap/Dockerfile → ...pper/examples/batchmap_flatmap/Dockerfile
@@ -1,19 +1,19 @@
 ####################################################################################################
 # base
 ####################################################################################################
-FROM alpine:3.12.3 as base
+FROM alpine:3.20 AS base
+ARG TARGETARCH
 RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata
 
-COPY dist/batchmap-flatmap /bin/batchmap-flatmap
+COPY dist/batchmap-flatmap-${TARGETARCH} /bin/batchmap-flatmap
 RUN chmod +x /bin/batchmap-flatmap
 
 ####################################################################################################
 # batchmap-flatmap
 ####################################################################################################
-FROM scratch as batch-flatmap
-ARG ARCH
+FROM scratch AS batch-flatmap
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=base /bin/batchmap-flatmap /bin/batchmap-flatmap

diff --git a/...mapper/examples/batchmap-flatmap/Makefile → ...mapper/examples/batchmap_flatmap/Makefile b/...mapper/examples/batchmap-flatmap/Makefile → ...mapper/examples/batchmap_flatmap/Makefile
@@ -1,10 +1,13 @@
 TAG ?= stable
 PUSH ?= false
 IMAGE_REGISTRY = quay.io/numaio/numaflow-go/batch-map-flatmap:${TAG}
+ARCHITECTURES = amd64 arm64
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/batchmap-flatmap main.go
+	for arch in $(ARCHITECTURES); do \
+		GOOS=linux GOARCH=$${arch} go build -v -o ./dist/batchmap-flatmap-$${arch} main.go; \
+	done
 
 .PHONY: image-push
 image-push: build

diff --git a/...apper/examples/batchmap-flatmap/README.md → ...apper/examples/batchmap_flatmap/README.md b/...apper/examples/batchmap-flatmap/README.md → ...apper/examples/batchmap_flatmap/README.md
diff --git a/...chmapper/examples/batchmap-flatmap/go.mod → ...chmapper/examples/batchmap_flatmap/go.mod b/...chmapper/examples/batchmap-flatmap/go.mod → ...chmapper/examples/batchmap_flatmap/go.mod
@@ -1,6 +1,6 @@
 module batchmap-flatmap
 
-go 1.22
+go 1.20
 
 replace github.com/numaproj/numaflow-go => ../../../..
 

diff --git a/...chmapper/examples/batchmap-flatmap/go.sum → ...chmapper/examples/batchmap_flatmap/go.sum b/...chmapper/examples/batchmap-flatmap/go.sum → ...chmapper/examples/batchmap_flatmap/go.sum
diff --git a/...hmapper/examples/batchmap-flatmap/main.go → ...hmapper/examples/batchmap_flatmap/main.go b/...hmapper/examples/batchmap-flatmap/main.go → ...hmapper/examples/batchmap_flatmap/main.go
diff --git a/...r/examples/batchmap-flatmap/pipeline.yaml → ...r/examples/batchmap_flatmap/pipeline.yaml b/...r/examples/batchmap-flatmap/pipeline.yaml → ...r/examples/batchmap_flatmap/pipeline.yaml
diff --git a/pkg/mapper/examples/even_odd/Dockerfile b/pkg/mapper/examples/even_odd/Dockerfile
@@ -1,19 +1,19 @@
 ####################################################################################################
 # base
 ####################################################################################################
-FROM alpine:3.12.3 as base
+FROM alpine:3.20 AS base
+ARG TARGETARCH
 RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata
 
-COPY dist/even-odd-example /bin/even-odd-example
+COPY dist/even-odd-example-${TARGETARCH} /bin/even-odd-example
 RUN chmod +x /bin/even-odd-example
 
 ####################################################################################################
 # even-odd
 ####################################################################################################
-FROM scratch as even-odd
-ARG ARCH
+FROM scratch AS even-odd
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=base /bin/even-odd-example /bin/even-odd-example

diff --git a/pkg/mapper/examples/even_odd/Makefile b/pkg/mapper/examples/even_odd/Makefile
@@ -1,10 +1,13 @@
 TAG ?= stable
 PUSH ?= false
 IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-even-odd:${TAG}
+ARCHITECTURES = amd64 arm64
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/even-odd-example main.go
+	for arch in $(ARCHITECTURES); do \
+		GOOS=linux GOARCH=$${arch} go build -v -o ./dist/even-odd-example-$${arch} main.go; \
+	done
 
 .PHONY: image-push
 image-push: build

diff --git a/pkg/mapper/examples/flatmap/Dockerfile b/pkg/mapper/examples/flatmap/Dockerfile
@@ -1,19 +1,19 @@
 ####################################################################################################
 # base
 ####################################################################################################
-FROM alpine:3.12.3 as base
+FROM alpine:3.20 AS base
+ARG TARGETARCH
 RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata
 
-COPY dist/flatmap-example /bin/flatmap-example
+COPY dist/flatmap-example-${TARGETARCH} /bin/flatmap-example
 RUN chmod +x /bin/flatmap-example
 
 ####################################################################################################
 # flatmap
 ####################################################################################################
-FROM scratch as flatmap
-ARG ARCH
+FROM scratch AS flatmap
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=base /bin/flatmap-example /bin/flatmap-example

diff --git a/pkg/mapper/examples/flatmap/Makefile b/pkg/mapper/examples/flatmap/Makefile
@@ -1,10 +1,13 @@
 TAG ?= stable
 PUSH ?= false
 IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-flatmap:${TAG}
+ARCHITECTURES = amd64 arm64
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/flatmap-example main.go
+	for arch in $(ARCHITECTURES); do \
+		GOOS=linux GOARCH=$${arch} go build -v -o ./dist/flatmap-example-$${arch} main.go; \
+	done
 
 .PHONY: image-push
 image-push: build

diff --git a/pkg/mapper/examples/forward_message/Dockerfile b/pkg/mapper/examples/forward_message/Dockerfile
@@ -1,19 +1,19 @@
 ####################################################################################################
 # base
 ####################################################################################################
-FROM alpine:3.12.3 as base
+FROM alpine:3.20 AS base
+ARG TARGETARCH
 RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata
 
-COPY dist/forward-message-example /bin/forward-message-example
+COPY dist/forward-message-example-${TARGETARCH} /bin/forward-message-example
 RUN chmod +x /bin/forward-message-example
 
 ####################################################################################################
 # forward-message
 ####################################################################################################
-FROM scratch as forward-message
-ARG ARCH
+FROM scratch AS forward-message
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=base /bin/forward-message-example /bin/forward-message-example

diff --git a/pkg/mapper/examples/forward_message/Makefile b/pkg/mapper/examples/forward_message/Makefile
@@ -1,10 +1,13 @@
 TAG ?= stable
 PUSH ?= false
 IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-forward-message:${TAG}
+ARCHITECTURES = amd64 arm64
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/forward-message-example main.go
+	for arch in $(ARCHITECTURES); do \
+		GOOS=linux GOARCH=$${arch} go build -v -o ./dist/forward-message-example-$${arch} main.go; \
+	done
 
 .PHONY: image-push
 image-push: build

diff --git a/pkg/mapper/examples/retry/Dockerfile b/pkg/mapper/examples/retry/Dockerfile
@@ -1,19 +1,19 @@
 ####################################################################################################
 # base
 ####################################################################################################
-FROM alpine:3.12.3 as base
+FROM alpine:3.20 AS base
+ARG TARGETARCH
 RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata
 
-COPY dist/retry /bin/retry
+COPY dist/retry-${TARGETARCH} /bin/retry
 RUN chmod +x /bin/retry
 
 ####################################################################################################
 # retry
 ####################################################################################################
-FROM scratch as retry
-ARG ARCH
+FROM scratch AS retry
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=base /bin/retry /bin/retry

diff --git a/pkg/mapper/examples/retry/Makefile b/pkg/mapper/examples/retry/Makefile
@@ -1,10 +1,13 @@
 TAG ?= stable
 PUSH ?= false
 IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-retry:${TAG}
+ARCHITECTURES = amd64 arm64
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/retry main.go
+	for arch in $(ARCHITECTURES); do \
+		GOOS=linux GOARCH=$${arch} go build -v -o ./dist/retry-$${arch} main.go; \
+	done
 
 .PHONY: image-push
 image-push: build

diff --git a/pkg/mapper/examples/tickgen/Dockerfile b/pkg/mapper/examples/tickgen/Dockerfile
@@ -1,19 +1,19 @@
 ####################################################################################################
 # base
 ####################################################################################################
-FROM alpine:3.12.3 as base
+FROM alpine:3.20 AS base
+ARG TARGETARCH
 RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata
 
-COPY dist/tickgen-example /bin/tickgen-example
+COPY dist/tickgen-example-${TARGETARCH} /bin/tickgen-example
 RUN chmod +x /bin/tickgen-example
 
 ####################################################################################################
 # flatmap
 ####################################################################################################
-FROM scratch as tickgen
-ARG ARCH
+FROM scratch AS tickgen
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=base /bin/tickgen-example /bin/tickgen-example

diff --git a/pkg/mapper/examples/tickgen/Makefile b/pkg/mapper/examples/tickgen/Makefile
@@ -1,10 +1,13 @@
 TAG ?= stable
 PUSH ?= false
 IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-tickgen:${TAG}
+ARCHITECTURES = amd64 arm64
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/tickgen-example main.go
+	for arch in $(ARCHITECTURES); do \
+		GOOS=linux GOARCH=$${arch} go build -v -o ./dist/tickgen-example-$${arch} main.go; \
+	done
 
 .PHONY: image-push
 image-push: build

diff --git a/pkg/mapstreamer/examples/flatmap_stream/Dockerfile b/pkg/mapstreamer/examples/flatmap_stream/Dockerfile
@@ -1,19 +1,19 @@
 ####################################################################################################
 # base
 ####################################################################################################
-FROM alpine:3.12.3 as base
+FROM alpine:3.20 AS base
+ARG TARGETARCH
 RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata
 
-COPY dist/flatmap-stream-example /bin/flatmap-stream-example
+COPY dist/flatmap-stream-example-${TARGETARCH} /bin/flatmap-stream-example
 RUN chmod +x /bin/flatmap-stream-example
 
 ####################################################################################################
 # flatmap
 ####################################################################################################
-FROM scratch as flatmap_stream
-ARG ARCH
+FROM scratch AS flatmap_stream
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=base /bin/flatmap-stream-example /bin/flatmap-stream-example

diff --git a/pkg/mapstreamer/examples/flatmap_stream/Makefile b/pkg/mapstreamer/examples/flatmap_stream/Makefile
@@ -1,10 +1,13 @@
 TAG ?= stable
 PUSH ?= false
 IMAGE_REGISTRY = quay.io/numaio/numaflow-go/map-flatmap-stream:${TAG}
+ARCHITECTURES = amd64 arm64
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/flatmap-stream-example main.go
+	for arch in $(ARCHITECTURES); do \
+		GOOS=linux GOARCH=$${arch} go build -v -o ./dist/flatmap-stream-example-$${arch} main.go; \
+	done
 
 .PHONY: image-push
 image-push: build

diff --git a/pkg/reducer/examples/counter/Dockerfile b/pkg/reducer/examples/counter/Dockerfile
@@ -1,19 +1,19 @@
 ####################################################################################################
 # base
 ####################################################################################################
-FROM alpine:3.12.3 as base
+FROM alpine:3.20 AS base
+ARG TARGETARCH
 RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata
 
-COPY dist/counter-example /bin/counter-example
+COPY dist/counter-example-${TARGETARCH} /bin/counter-example
 RUN chmod +x /bin/counter-example
 
 ####################################################################################################
 # counter
 ####################################################################################################
-FROM scratch as counter
-ARG ARCH
+FROM scratch AS counter
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=base /bin/counter-example /bin/counter-example

diff --git a/pkg/reducer/examples/counter/Makefile b/pkg/reducer/examples/counter/Makefile
@@ -1,10 +1,13 @@
 TAG ?= stable
 PUSH ?= false
 IMAGE_REGISTRY = quay.io/numaio/numaflow-go/reduce-counter:${TAG}
+ARCHITECTURES = amd64 arm64
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/counter-example main.go
+	for arch in $(ARCHITECTURES); do \
+		GOOS=linux GOARCH=$${arch} go build -v -o ./dist/counter-example-$${arch} main.go; \
+	done
 
 .PHONY: image-push
 image-push: build

diff --git a/pkg/reducer/examples/sum/Dockerfile b/pkg/reducer/examples/sum/Dockerfile
@@ -1,19 +1,19 @@
 ####################################################################################################
 # base
 ####################################################################################################
-FROM alpine:3.12.3 as base
+FROM alpine:3.20 AS base
+ARG TARGETARCH
 RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata
 
-COPY dist/sum-example /bin/sum-example
+COPY dist/sum-example-${TARGETARCH} /bin/sum-example
 RUN chmod +x /bin/sum-example
 
 ####################################################################################################
 # sum
 ####################################################################################################
-FROM scratch as sum
-ARG ARCH
+FROM scratch AS sum
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=base /bin/sum-example /bin/sum-example

diff --git a/pkg/reducer/examples/sum/Makefile b/pkg/reducer/examples/sum/Makefile
@@ -1,10 +1,13 @@
 TAG ?= stable
 PUSH ?= false
 IMAGE_REGISTRY = quay.io/numaio/numaflow-go/reduce-sum:${TAG}
+ARCHITECTURES = amd64 arm64
 
 .PHONY: build
 build:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sum-example main.go
+	for arch in $(ARCHITECTURES); do \
+		GOOS=linux GOARCH=$${arch} go build -v -o ./dist/sum-example-$${arch} main.go; \
+	done
 
 .PHONY: image-push
 image-push: build

diff --git a/pkg/reducestreamer/examples/counter/Dockerfile b/pkg/reducestreamer/examples/counter/Dockerfile
@@ -1,19 +1,19 @@
 ####################################################################################################
 # base
 ####################################################################################################
-FROM alpine:3.12.3 as base
+FROM alpine:3.20 AS base
+ARG TARGETARCH
 RUN apk update && apk upgrade && \
     apk add ca-certificates && \
     apk --no-cache add tzdata
 
-COPY dist/counter-example /bin/counter-example
+COPY dist/counter-example-${TARGETARCH} /bin/counter-example
 RUN chmod +x /bin/counter-example
 
 ####################################################################################################
 # counter
 ####################################################################################################
-FROM scratch as counter
-ARG ARCH
+FROM scratch AS counter
 COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
 COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 COPY --from=base /bin/counter-example /bin/counter-example