From fb21149ee351650e76da8a7603bd271849d5451f Mon Sep 17 00:00:00 2001 From: Sidhant Kohli Date: Mon, 8 Jul 2024 16:16:44 -0700 Subject: [PATCH] refactor Signed-off-by: Sidhant Kohli --- .../examples/batchmap-flatmap/Dockerfile | 4 ++-- .../examples/batchmap-flatmap/Makefile | 4 ++-- .../examples/batchmap-flatmap/README.md | 16 ++++++++++++++++ .../examples/batchmap-flatmap/pipeline.yaml | 2 +- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/pkg/batchmapper/examples/batchmap-flatmap/Dockerfile b/pkg/batchmapper/examples/batchmap-flatmap/Dockerfile index 98468775..855508a0 100644 --- a/pkg/batchmapper/examples/batchmap-flatmap/Dockerfile +++ b/pkg/batchmapper/examples/batchmap-flatmap/Dockerfile @@ -10,9 +10,9 @@ COPY dist/batchmap-flatmap /bin/batchmap-flatmap RUN chmod +x /bin/batchmap-flatmap #################################################################################################### -# flatmap +# batchmap-flatmap #################################################################################################### -FROM scratch as flatmap +FROM scratch as batch-flatmap ARG ARCH COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt diff --git a/pkg/batchmapper/examples/batchmap-flatmap/Makefile b/pkg/batchmapper/examples/batchmap-flatmap/Makefile index 0984106d..ea6b9992 100644 --- a/pkg/batchmapper/examples/batchmap-flatmap/Makefile +++ b/pkg/batchmapper/examples/batchmap-flatmap/Makefile @@ -8,11 +8,11 @@ build: .PHONY: image-push image-push: build - docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target flatmap . --push + docker buildx build -t ${IMAGE_REGISTRY} --platform linux/amd64,linux/arm64 --target batch-flatmap . --push .PHONY: image image: build - docker build -t ${IMAGE_REGISTRY} --target flatmap . + docker build -t ${IMAGE_REGISTRY} --target batch-flatmap . @if [ "$(PUSH)" = "true" ]; then docker push ${IMAGE_REGISTRY}; fi clean: diff --git a/pkg/batchmapper/examples/batchmap-flatmap/README.md b/pkg/batchmapper/examples/batchmap-flatmap/README.md index 093e9030..135b5ecb 100644 --- a/pkg/batchmapper/examples/batchmap-flatmap/README.md +++ b/pkg/batchmapper/examples/batchmap-flatmap/README.md @@ -3,6 +3,22 @@ An example User Defined Function that demonstrates how to write a batch map based `flatmap` User Defined Function. +To start a vertex in batch map mode we need to add the annotations as following +```yaml + - name: go-split + metadata: + annotations: + numaflow.numaproj.io/batch-map: "true" + scale: + min: 1 + udf: + container: + # Split input message into an array with comma, see https://github.com/numaproj/numaflow-go/tree/main/pkg/batchmapper/examples/batchmap-flatmap + image: quay.io/numaio/numaflow-go/batch-map-flatmap:stable + imagePullPolicy: Always +``` + + Some considerations for batch map are as follows - The user will have to ensure that the BatchResponse is tagged with the correct request ID as this will be used by Numaflow for populating information required for system correctness like MessageID for the ISB deduplication. diff --git a/pkg/batchmapper/examples/batchmap-flatmap/pipeline.yaml b/pkg/batchmapper/examples/batchmap-flatmap/pipeline.yaml index 7b7cd3b9..7bac7625 100644 --- a/pkg/batchmapper/examples/batchmap-flatmap/pipeline.yaml +++ b/pkg/batchmapper/examples/batchmap-flatmap/pipeline.yaml @@ -15,7 +15,7 @@ spec: min: 1 udf: container: - # Split input message into an array with comma, see https://github.com/numaproj/numaflow-go/tree/main/pkg/mapper/examples/flatmap + # Split input message into an array with comma, see https://github.com/numaproj/numaflow-go/tree/main/pkg/batchmapper/examples/batchmap-flatmap image: quay.io/numaio/numaflow-go/batch-map-flatmap:stable imagePullPolicy: Always - name: go-udsink