diff --git a/.github/workflows/build-push.yaml b/.github/workflows/build-push.yaml index b2fbe938..9ada4718 100644 --- a/.github/workflows/build-push.yaml +++ b/.github/workflows/build-push.yaml @@ -21,9 +21,11 @@ jobs: "pkg/mapper/examples/retry", "pkg/mapper/examples/tickgen", "pkg/mapstreamer/examples/flatmap_stream", "pkg/reducer/examples/counter", "pkg/reducer/examples/sum", "pkg/reducestreamer/examples/counter", "pkg/reducestreamer/examples/sum", "pkg/sessionreducer/examples/counter", "pkg/sessionreducer/examples/sum", - "pkg/sideinput/examples/simple-sideinput/udf", "pkg/sideinput/examples/simple-sideinput", "pkg/sinker/examples/log", + "pkg/sideinput/examples/simple_sideinput/udf", "pkg/sideinput/examples/simple_sideinput", "pkg/sinker/examples/log", "pkg/sourcer/examples/simple_source", "pkg/sourcetransformer/examples/assign_event_time", "pkg/sourcetransformer/examples/event_time_filter", - "pkg/sinker/examples/fallback" + "pkg/sinker/examples/fallback", "pkg/sideinput/examples/map_sideinput", "pkg/sideinput/examples/reduce_sideinput", + "pkg/sideinput/examples/sideinput_function", "pkg/sideinput/examples/simple_source_with_sideinput", + "pkg/sideinput/examples/sink_sideinput", "pkg/sinker/examples/redis-sink" ] steps: diff --git a/pkg/mapper/examples/forward_message/go.mod b/pkg/mapper/examples/forward_message/go.mod index 1c93fc43..a03e9712 100644 --- a/pkg/mapper/examples/forward_message/go.mod +++ b/pkg/mapper/examples/forward_message/go.mod @@ -1,4 +1,4 @@ -module even_odd +module forward_message go 1.20 diff --git a/pkg/sessionreducer/examples/sum/go.mod b/pkg/sessionreducer/examples/sum/go.mod index f87d7f22..ce606391 100644 --- a/pkg/sessionreducer/examples/sum/go.mod +++ b/pkg/sessionreducer/examples/sum/go.mod @@ -1,4 +1,4 @@ -module counter +module sum go 1.20 diff --git a/pkg/sideinput/examples/map_sideinput/Dockerfile b/pkg/sideinput/examples/map_sideinput/Dockerfile new file mode 100644 index 00000000..a2129aba --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/map-sideinput-example /bin/map-sideinput-example +RUN chmod +x /bin/map-sideinput-example + +#################################################################################################### +# sideinput +#################################################################################################### +FROM scratch as sideinput +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/map-sideinput-example /bin/map-sideinput-example +ENTRYPOINT [ "/bin/map-sideinput-example" ] diff --git a/pkg/sideinput/examples/map_sideinput/Makefile b/pkg/sideinput/examples/map_sideinput/Makefile new file mode 100644 index 00000000..56fb208e --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/map-sideinput-example main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/map-sideinput:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/map-sideinput:${TAG}" --target sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-sideinput:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/map_sideinput/README.md b/pkg/sideinput/examples/map_sideinput/README.md new file mode 100644 index 00000000..139706a8 --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/README.md @@ -0,0 +1,49 @@ +# Map SideInput Example +An example that demonstrates how to write a [sideinput](https://numaflow.numaproj.io/user-guide/reference/side-inputs/) function along with a sample User Defined function which watches and used the corresponding side input with Mapper function. + +### SideInput +```golang +// handle is the side input handler function. +func handle(_ context.Context) sideinputsdk.Message { + // generate message based on even and odd counter + counter++ + if counter%2 == 0 { + return sideinputsdk.BroadcastMessage([]byte("even")) + } + // BroadcastMessage() is used to broadcast the message with the given value to other side input vertices. + // val must be converted to []byte. + return sideinputsdk.BroadcastMessage([]byte("odd")) +} +``` + +By using [UDF](#user-defined-function), User perform the retrieval/update for the side input value and perform some operation based via mapper function on it and broadcast/drop a message to other vertices. + +### User Defined Function +The UDF vertex will watch for changes to this file and whenever there is a change it will read the file to obtain the new side input value. + +### Pipeline spec +In the spec we need to define the side input vertex and the UDF vertex. The UDF vertex will have the side input vertex as a side input. + +```yaml +spec: + sideInputs: + - name: myticker + container: + image: "quay.io/numaio/numaflow-go/map-sideinput:stable" + imagePullPolicy: Always + trigger: + schedule: "@every 5s" +``` +Vertex spec for the UDF vertex: +```yaml + - name: si-e2e + udf: + container: + image: "quay.io/numaio/numaflow-go/map-sideinput-udf:stable" + imagePullPolicy: Always + sideInputs: + - myticker +``` + + + diff --git a/pkg/sideinput/examples/map_sideinput/go.mod b/pkg/sideinput/examples/map_sideinput/go.mod new file mode 100644 index 00000000..f7b020ad --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/go.mod @@ -0,0 +1,17 @@ +module map_sideinput + +go 1.21.2 + +replace github.com/numaproj/numaflow-go => ../../../.. + +require github.com/numaproj/numaflow-go v0.7.0-rc2 + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/sideinput/examples/map_sideinput/go.sum b/pkg/sideinput/examples/map_sideinput/go.sum new file mode 100644 index 00000000..917367e7 --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/go.sum @@ -0,0 +1,29 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/map_sideinput/main.go b/pkg/sideinput/examples/map_sideinput/main.go new file mode 100644 index 00000000..845422fb --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/main.go @@ -0,0 +1,45 @@ +/* +Copyright 2023 The Numaproj Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log" + + sideinputsdk "github.com/numaproj/numaflow-go/pkg/sideinput" +) + +var counter = 0 + +// handle is the side input handler function. +func handle(_ context.Context) sideinputsdk.Message { + // generate message based on even and odd counter + counter++ + if counter%2 == 0 { + return sideinputsdk.BroadcastMessage([]byte("even")) + } + // BroadcastMessage() is used to broadcast the message with the given value to other side input vertices. + // val must be converted to []byte. + return sideinputsdk.BroadcastMessage([]byte("odd")) +} + +func main() { + // Start the side input server. + err := sideinputsdk.NewSideInputServer(sideinputsdk.RetrieveFunc(handle)).Start(context.Background()) + if err != nil { + log.Panic("Failed to start side input server: ", err) + } +} diff --git a/pkg/sideinput/examples/map_sideinput/udf/Dockerfile b/pkg/sideinput/examples/map_sideinput/udf/Dockerfile new file mode 100644 index 00000000..108f0b63 --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/udf/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/map-sideinput-udf /bin/map-sideinput-udf +RUN chmod +x /bin/map-sideinput-udf + +#################################################################################################### +# udf-sideinput +#################################################################################################### +FROM scratch as sideinput-udf +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/map-sideinput-udf /bin/map-sideinput-udf +ENTRYPOINT [ "/bin/map-sideinput-udf" ] \ No newline at end of file diff --git a/pkg/sideinput/examples/map_sideinput/udf/Makefile b/pkg/sideinput/examples/map_sideinput/udf/Makefile new file mode 100644 index 00000000..f38019e7 --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/udf/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/map-sideinput-udf main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/map-sideinput-udf:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput-udf . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/map-sideinput-udf:${TAG}" --target sideinput-udf . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/map-sideinput-udf:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/map_sideinput/udf/go.mod b/pkg/sideinput/examples/map_sideinput/udf/go.mod new file mode 100644 index 00000000..46e87cb0 --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/udf/go.mod @@ -0,0 +1,20 @@ +module udf + +go 1.21.2 + +replace github.com/numaproj/numaflow-go => ../../../../.. + +require ( + github.com/fsnotify/fsnotify v1.6.0 + github.com/numaproj/numaflow-go v0.7.0-rc2 +) + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/sideinput/examples/map_sideinput/udf/go.sum b/pkg/sideinput/examples/map_sideinput/udf/go.sum new file mode 100644 index 00000000..dbcfb187 --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/udf/go.sum @@ -0,0 +1,32 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/map_sideinput/udf/main.go b/pkg/sideinput/examples/map_sideinput/udf/main.go new file mode 100644 index 00000000..88c5de00 --- /dev/null +++ b/pkg/sideinput/examples/map_sideinput/udf/main.go @@ -0,0 +1,109 @@ +/* +Copyright 2023 The Numaproj Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log" + "os" + "path" + "sync" + + "github.com/fsnotify/fsnotify" + + "github.com/numaproj/numaflow-go/pkg/mapper" + "github.com/numaproj/numaflow-go/pkg/sideinput" +) + +var sideInputName = "myticker" +var sideInputData []byte +var mu sync.RWMutex + +func mapFn(_ context.Context, _ []string, d mapper.Datum) mapper.Messages { + msg := d.Value() + + // take a read lock before updating sideInputData data to prevent race condition + mu.RLock() + siData := sideInputData + mu.RUnlock() + + if len(siData) > 0 { + if string(siData) == "even" { + return mapper.MessagesBuilder().Append(mapper.NewMessage([]byte(string(msg) + "-even-data"))) + } else if string(siData) == "odd" { + return mapper.MessagesBuilder().Append(mapper.NewMessage([]byte(string(msg) + "-odd-data"))) + } + } + + return mapper.MessagesBuilder().Append(mapper.MessageToDrop()) +} + +func main() { + // Create a new fsnotify watcher + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + + // Add a path to the watcher + err = watcher.Add(sideinput.DirPath) + if err != nil { + log.Fatal(err) + } + + // Start a goroutine to listen for events from the watcher + go fileWatcher(watcher, sideInputName) + + err = mapper.NewServer(mapper.MapperFunc(mapFn)).Start(context.Background()) + if err != nil { + log.Panic("Failed to start map function server: ", err) + } +} + +// fileWatcher will watch for any changes in side input file and set data in +// sideInputData global variable. +func fileWatcher(watcher *fsnotify.Watcher, sideInputName string) { + log.Println("Watching for changes in side input file: ", sideinput.DirPath) + p := path.Join(sideinput.DirPath, sideInputName) + for { + select { + case event, ok := <-watcher.Events: + if !ok { + log.Println("watcher.Events channel closed") + return + } + if event.Op&fsnotify.Create == fsnotify.Create && event.Name == p { + log.Println("Side input file has been created:", event.Name) + b, err := os.ReadFile(p) + if err != nil { + log.Println("Failed to read side input file: ", err) + } + + // take a lock before updating sideInputData data to prevent race condition + mu.Lock() + sideInputData = b[:] + mu.Unlock() + } + case err, ok := <-watcher.Errors: + if !ok { + log.Println("watcher.Errors channel closed") + return + } + log.Println("error:", err) + } + } +} diff --git a/pkg/sideinput/examples/reduce_sideinput/Dockerfile b/pkg/sideinput/examples/reduce_sideinput/Dockerfile new file mode 100644 index 00000000..ca830d0b --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/reduce-sideinput-example /bin/reduce-sideinput-example +RUN chmod +x /bin/reduce-sideinput-example + +#################################################################################################### +# sideinput +#################################################################################################### +FROM scratch as sideinput +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/reduce-sideinput-example /bin/reduce-sideinput-example +ENTRYPOINT [ "/bin/reduce-sideinput-example" ] diff --git a/pkg/sideinput/examples/reduce_sideinput/Makefile b/pkg/sideinput/examples/reduce_sideinput/Makefile new file mode 100644 index 00000000..43d85e38 --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/reduce-sideinput-example main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/reduce-sideinput:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/reduce-sideinput:${TAG}" --target sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-sideinput:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/reduce_sideinput/README.md b/pkg/sideinput/examples/reduce_sideinput/README.md new file mode 100644 index 00000000..ea433265 --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/README.md @@ -0,0 +1,54 @@ +# Reduce SideInput Example +An example that demonstrates how to write a [sideinput](https://numaflow.numaproj.io/user-guide/reference/side-inputs/) function along with a sample User Defined function which watches and used the corresponding side input with Mapper function. + +### SideInput +```golang +// handle is the side input handler function. +func handle(_ context.Context) sideinputsdk.Message { + counter++ + // BroadcastMessage() is used to broadcast the message with the given value to other side input vertices. + // val must be converted to []byte. + return sideinputsdk.BroadcastMessage([]byte(strconv.Itoa(counter))) +} +``` + +By using [UDF](#user-defined-function), User perform the retrieval/update for the side input value and perform some operation based via reducer function on it and broadcast/drop a message to other vertices. + +### User Defined Function +The UDF vertex will watch for changes to this file and whenever there is a change it will read the file to obtain the new side input value. + +### Pipeline spec +In the spec we need to define the side input vertex and the UDF vertex. The UDF vertex will have the side input vertex as a side input. + +```yaml +spec: + sideInputs: + - name: myticker + container: + image: "quay.io/numaio/numaflow-go/reduce-sideinput:stable" + imagePullPolicy: Always + trigger: + schedule: "@every 5s" +``` +Vertex spec for the UDF vertex: +```yaml + - name: si-e2e + udf: + container: + image: "quay.io/numaio/numaproj-contrib/reduce-sideinput-udf:stable" + imagePullPolicy: Always + groupBy: + window: + fixed: + length: 10s + keyed: true + storage: + persistentVolumeClaim: + volumeSize: 10Gi + accessMode: ReadWriteOnce + sideInputs: + - myticker +``` + + + diff --git a/pkg/sideinput/examples/reduce_sideinput/go.mod b/pkg/sideinput/examples/reduce_sideinput/go.mod new file mode 100644 index 00000000..570247ef --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/go.mod @@ -0,0 +1,17 @@ +module reduce_sideinput + +go 1.21 + +replace github.com/numaproj/numaflow-go => ../../../.. + +require github.com/numaproj/numaflow-go v0.7.0-rc2 + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/sideinput/examples/reduce_sideinput/go.sum b/pkg/sideinput/examples/reduce_sideinput/go.sum new file mode 100644 index 00000000..917367e7 --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/go.sum @@ -0,0 +1,29 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/reduce_sideinput/main.go b/pkg/sideinput/examples/reduce_sideinput/main.go new file mode 100644 index 00000000..1364b797 --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/main.go @@ -0,0 +1,42 @@ +/* +Copyright 2023 The Numaproj Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log" + "strconv" + + sideinputsdk "github.com/numaproj/numaflow-go/pkg/sideinput" +) + +var counter = 0 + +// handle is the side input handler function. +func handle(_ context.Context) sideinputsdk.Message { + counter++ + // BroadcastMessage() is used to broadcast the message with the given value to other side input vertices. + // val must be converted to []byte. + return sideinputsdk.BroadcastMessage([]byte(strconv.Itoa(counter))) +} + +func main() { + // Start the side input server. + err := sideinputsdk.NewSideInputServer(sideinputsdk.RetrieveFunc(handle)).Start(context.Background()) + if err != nil { + log.Panic("Failed to start side input server: ", err) + } +} diff --git a/pkg/sideinput/examples/reduce_sideinput/udf/Dockerfile b/pkg/sideinput/examples/reduce_sideinput/udf/Dockerfile new file mode 100644 index 00000000..7ae826ea --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/udf/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/reduce-sideinput-udf /bin/reduce-sideinput-udf +RUN chmod +x /bin/reduce_sideinput-udf + +#################################################################################################### +# udf-sideinput +#################################################################################################### +FROM scratch as sideinput-udf +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/reduce-sideinput-udf /bin/reduce-sideinput-udf +ENTRYPOINT [ "/bin/reduce-sideinput-udf" ] \ No newline at end of file diff --git a/pkg/sideinput/examples/reduce_sideinput/udf/Makefile b/pkg/sideinput/examples/reduce_sideinput/udf/Makefile new file mode 100644 index 00000000..12336203 --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/udf/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/reduce-sideinput-udf main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/reduce-sideinput-udf:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput-udf . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/reduce-sideinput-udf:${TAG}" --target sideinput-udf . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/reduce-sideinput-udf:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/reduce_sideinput/udf/go.mod b/pkg/sideinput/examples/reduce_sideinput/udf/go.mod new file mode 100644 index 00000000..b6b1d6dd --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/udf/go.mod @@ -0,0 +1,21 @@ +module udf + +go 1.21.2 + +replace github.com/numaproj/numaflow-go => ../../../../.. + +require ( + github.com/fsnotify/fsnotify v1.7.0 + github.com/numaproj/numaflow-go v0.7.0-rc2 +) + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/sideinput/examples/reduce_sideinput/udf/go.sum b/pkg/sideinput/examples/reduce_sideinput/udf/go.sum new file mode 100644 index 00000000..18817608 --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/udf/go.sum @@ -0,0 +1,33 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/reduce_sideinput/udf/main.go b/pkg/sideinput/examples/reduce_sideinput/udf/main.go new file mode 100644 index 00000000..0d11b6d2 --- /dev/null +++ b/pkg/sideinput/examples/reduce_sideinput/udf/main.go @@ -0,0 +1,128 @@ +/* +Copyright 2023 The Numaproj Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log" + "os" + "path" + "strconv" + "sync" + + "github.com/fsnotify/fsnotify" + + "github.com/numaproj/numaflow-go/pkg/reducer" + "github.com/numaproj/numaflow-go/pkg/sideinput" +) + +// SumReducerCreator implements the reducer.ReducerCreator interface which creates a reducer +type SumReducerCreator struct { +} + +func (s *SumReducerCreator) Create() reducer.Reducer { + return &Sum{} +} + +// Sum is a reducer that sum up the values for the given keys +type Sum struct { + sum int +} + +var sideInputName = "myticker" +var sideInputData []byte +var mu sync.RWMutex + +func (s *Sum) Reduce(ctx context.Context, keys []string, reduceCh <-chan reducer.Datum, md reducer.Metadata) reducer.Messages { + mu.RLock() + siData := sideInputData + mu.RUnlock() + + // sum up the values + for d := range reduceCh { + value, err := strconv.Atoi(string(d.Value())) + if err != nil { + log.Printf("unable to convert the value to integer: %v\n", err) + continue + } + + s.sum += value + } + + if s.sum > 0 && len(siData) > 0 { + msg := []byte("reduce_sideinput") + return reducer.MessagesBuilder().Append(reducer.NewMessage(msg)) + } else { + return reducer.MessagesBuilder().Append(reducer.MessageToDrop()) + } +} + +func main() { + // Create a new fsnotify watcher + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + + // Add a path to the watcher + err = watcher.Add(sideinput.DirPath) + if err != nil { + log.Fatal(err) + } + + // Start a goroutine to listen for events from the watcher + go fileWatcher(watcher, sideInputName) + + err = reducer.NewServer(&SumReducerCreator{}).Start(context.Background()) + if err != nil { + log.Panic("Failed to start reducer function server: ", err) + } +} + +// fileWatcher will watch for any changes in side input file and set data in +// sideInputData global variable. +func fileWatcher(watcher *fsnotify.Watcher, sideInputName string) { + log.Println("Watching for changes in side input file: ", sideinput.DirPath) + p := path.Join(sideinput.DirPath, sideInputName) + for { + select { + case event, ok := <-watcher.Events: + if !ok { + log.Println("watcher.Events channel closed") + return + } + if event.Op&fsnotify.Create == fsnotify.Create && event.Name == p { + log.Println("Side input file has been created:", event.Name) + b, err := os.ReadFile(p) + if err != nil { + log.Println("Failed to read side input file: ", err) + } + + // take a lock before updating sideInputData data to prevent race condition + mu.Lock() + sideInputData = b[:] + mu.Unlock() + } + case err, ok := <-watcher.Errors: + if !ok { + log.Println("watcher.Errors channel closed") + return + } + log.Println("error:", err) + } + } +} diff --git a/pkg/sideinput/examples/sideinput_function/Dockerfile b/pkg/sideinput/examples/sideinput_function/Dockerfile new file mode 100644 index 00000000..31c55a31 --- /dev/null +++ b/pkg/sideinput/examples/sideinput_function/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/sideinput-function /bin/sideinput-function +RUN chmod +x /bin/sideinput-function + +#################################################################################################### +# sideinput +#################################################################################################### +FROM scratch as sideinput +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/sideinput-function /bin/sideinput-function +ENTRYPOINT [ "/bin/sideinput-function" ] \ No newline at end of file diff --git a/pkg/sideinput/examples/sideinput_function/Makefile b/pkg/sideinput/examples/sideinput_function/Makefile new file mode 100644 index 00000000..c6187577 --- /dev/null +++ b/pkg/sideinput/examples/sideinput_function/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/sideinput-function main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/sideinput-function:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/sideinput-function:${TAG}" --target sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/sideinput-function:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/sideinput_function/README.md b/pkg/sideinput/examples/sideinput_function/README.md new file mode 100644 index 00000000..b722e798 --- /dev/null +++ b/pkg/sideinput/examples/sideinput_function/README.md @@ -0,0 +1,77 @@ +# SideInput Example + +An example that demonstrates how to write a `sideinput` function along with a sample `User Defined function` +which watches and used the corresponding side input. + +### SideInput +```golang +// handle is the side input handler function. +func handle(_ context.Context) sideinputsdk.Message { + t := time.Now() + /*This code is a simple side input handler that broadcasts a + message based on an internal counter. The counter increments with every call, + and the program broadcasts either "e2e-even" or "e2e-odd" depending on the current value of the counter. + + */ + + // BroadcastMessage() is used to broadcast the message with the given value to other side input vertices. + // val must be converted to []byte. + if rand.Int()%2 == 0 { + return sideinputsdk.BroadcastMessage([]byte(`e2e-even`)) + } + + return sideinputsdk.BroadcastMessage([]byte(`e2e-odd`)) +} +``` +After performing the retrieval/update for the side input value the user can choose to either broadcast the +message to other side input vertices or drop the message. The side input message is not retried. + +For each side input there will be a file with the given path and after any update to the side input value the file will +be updated. + +The directory is fixed and can be accessed through sideinput constants `sideinput.DirPath`. +The file name is the name of the side input. +```golang +sideinput.DirPath -> "/var/numaflow/side-inputs" +sideInputFileName -> "/var/numaflow/side-inputs/sideInputName" +``` + +### User Defined Function + +The UDF vertex will watch for changes to this file and whenever there is a change it will read the file to obtain the new side input value. + + +### Pipeline spec + +In the spec we need to define the side input vertex and the UDF vertex. The UDF vertex will have the side input vertex as a side input. + +Side input spec: +```yaml +spec: + sideInputs: + - name: myticker + container: + image: "quay.io/numaio/numaflow-go/sideinput-function:stable" + imagePullPolicy: Always + trigger: + schedule: "*/2 * * * *" + +``` + +Vertex spec for the UDF vertex: +```yaml + - name: si-log + udf: + container: + image: "quay.io/numaio/numaflow-go/sideinput-function-udf:stable" + imagePullPolicy: Always + containerTemplate: + env: + - name: NUMAFLOW_DEBUG + value: "true" # DO NOT forget the double quotes!!! + sideInputs: + - myticker +``` + + + diff --git a/pkg/sideinput/examples/sideinput_function/go.mod b/pkg/sideinput/examples/sideinput_function/go.mod new file mode 100644 index 00000000..d2b882d8 --- /dev/null +++ b/pkg/sideinput/examples/sideinput_function/go.mod @@ -0,0 +1,17 @@ +module even_odd_sideinput + +go 1.21 + +replace github.com/numaproj/numaflow-go => ../../../.. + +require github.com/numaproj/numaflow-go v0.7.0-rc2 + +require ( + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/sideinput/examples/sideinput_function/go.sum b/pkg/sideinput/examples/sideinput_function/go.sum new file mode 100644 index 00000000..917367e7 --- /dev/null +++ b/pkg/sideinput/examples/sideinput_function/go.sum @@ -0,0 +1,29 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/sideinput_function/main.go b/pkg/sideinput/examples/sideinput_function/main.go new file mode 100644 index 00000000..5f1dae0c --- /dev/null +++ b/pkg/sideinput/examples/sideinput_function/main.go @@ -0,0 +1,29 @@ +package main + +import ( + "context" + sideinputsdk "github.com/numaproj/numaflow-go/pkg/sideinput" + "log" +) + +var counter = 0 + +// handle is the side input handler function. +func handle(_ context.Context) sideinputsdk.Message { + // BroadcastMessage() is used to broadcast the message with the given value to other side input vertices. + // val must be converted to []byte. + counter = (counter + 1) % 10 + if counter%2 == 0 { + return sideinputsdk.BroadcastMessage([]byte(`e2e-even`)) + } + + return sideinputsdk.BroadcastMessage([]byte(`e2e-odd`)) +} + +func main() { + // Start the side input server. + err := sideinputsdk.NewSideInputServer(sideinputsdk.RetrieveFunc(handle)).Start(context.Background()) + if err != nil { + log.Panic("Failed to start side input server: ", err) + } +} diff --git a/pkg/sideinput/examples/simple-sideinput/Dockerfile b/pkg/sideinput/examples/simple_sideinput/Dockerfile similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/Dockerfile rename to pkg/sideinput/examples/simple_sideinput/Dockerfile diff --git a/pkg/sideinput/examples/simple-sideinput/Makefile b/pkg/sideinput/examples/simple_sideinput/Makefile similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/Makefile rename to pkg/sideinput/examples/simple_sideinput/Makefile diff --git a/pkg/sideinput/examples/simple-sideinput/README.md b/pkg/sideinput/examples/simple_sideinput/README.md similarity index 94% rename from pkg/sideinput/examples/simple-sideinput/README.md rename to pkg/sideinput/examples/simple_sideinput/README.md index c80afb5d..93029dbf 100644 --- a/pkg/sideinput/examples/simple-sideinput/README.md +++ b/pkg/sideinput/examples/simple_sideinput/README.md @@ -49,7 +49,7 @@ spec: sideInputs: - name: myticker container: - image: "quay.io/numaio/numaflow-go/sideinput:v0.5.0" + image: "quay.io/numaio/numaflow-go/sideinput-example:v0.5.0" imagePullPolicy: Always trigger: schedule: "*/2 * * * *" @@ -61,7 +61,7 @@ Vertex spec for the UDF vertex: - name: si-log udf: container: - image: "quay.io/numaio/numaflow-go/udf-sideinput:v0.5.0" + image: "quay.io/numaio/numaflow-go/sideinput-example-udf:v0.5.0" imagePullPolicy: Always containerTemplate: env: diff --git a/pkg/sideinput/examples/simple-sideinput/go.mod b/pkg/sideinput/examples/simple_sideinput/go.mod similarity index 95% rename from pkg/sideinput/examples/simple-sideinput/go.mod rename to pkg/sideinput/examples/simple_sideinput/go.mod index 0a330912..2194a0b1 100644 --- a/pkg/sideinput/examples/simple-sideinput/go.mod +++ b/pkg/sideinput/examples/simple_sideinput/go.mod @@ -1,4 +1,4 @@ -module sideinput +module simple_sideinput go 1.20 diff --git a/pkg/sideinput/examples/simple-sideinput/go.sum b/pkg/sideinput/examples/simple_sideinput/go.sum similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/go.sum rename to pkg/sideinput/examples/simple_sideinput/go.sum diff --git a/pkg/sideinput/examples/simple-sideinput/main.go b/pkg/sideinput/examples/simple_sideinput/main.go similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/main.go rename to pkg/sideinput/examples/simple_sideinput/main.go diff --git a/pkg/sideinput/examples/simple-sideinput/pipeline.yaml b/pkg/sideinput/examples/simple_sideinput/pipeline.yaml similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/pipeline.yaml rename to pkg/sideinput/examples/simple_sideinput/pipeline.yaml diff --git a/pkg/sideinput/examples/simple-sideinput/udf/Dockerfile b/pkg/sideinput/examples/simple_sideinput/udf/Dockerfile similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/udf/Dockerfile rename to pkg/sideinput/examples/simple_sideinput/udf/Dockerfile diff --git a/pkg/sideinput/examples/simple-sideinput/udf/Makefile b/pkg/sideinput/examples/simple_sideinput/udf/Makefile similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/udf/Makefile rename to pkg/sideinput/examples/simple_sideinput/udf/Makefile diff --git a/pkg/sideinput/examples/simple-sideinput/udf/go.mod b/pkg/sideinput/examples/simple_sideinput/udf/go.mod similarity index 96% rename from pkg/sideinput/examples/simple-sideinput/udf/go.mod rename to pkg/sideinput/examples/simple_sideinput/udf/go.mod index 038c1f3e..c6510bb5 100644 --- a/pkg/sideinput/examples/simple-sideinput/udf/go.mod +++ b/pkg/sideinput/examples/simple_sideinput/udf/go.mod @@ -1,4 +1,4 @@ -module even_odd +module udf go 1.20 diff --git a/pkg/sideinput/examples/simple-sideinput/udf/go.sum b/pkg/sideinput/examples/simple_sideinput/udf/go.sum similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/udf/go.sum rename to pkg/sideinput/examples/simple_sideinput/udf/go.sum diff --git a/pkg/sideinput/examples/simple-sideinput/udf/main.go b/pkg/sideinput/examples/simple_sideinput/udf/main.go similarity index 100% rename from pkg/sideinput/examples/simple-sideinput/udf/main.go rename to pkg/sideinput/examples/simple_sideinput/udf/main.go diff --git a/pkg/sideinput/examples/simple_source_with_sideinput/Dockerfile b/pkg/sideinput/examples/simple_source_with_sideinput/Dockerfile new file mode 100644 index 00000000..ba4e9c66 --- /dev/null +++ b/pkg/sideinput/examples/simple_source_with_sideinput/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/simple-source-example /bin/simple-source-example +RUN chmod +x /bin/simple-source-example + +#################################################################################################### +# simple-source +#################################################################################################### +FROM scratch as sideinput +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/simple-source-example /bin/simple-source-example +ENTRYPOINT [ "/bin/simple-source-example" ] diff --git a/pkg/sideinput/examples/simple_source_with_sideinput/Makefile b/pkg/sideinput/examples/simple_source_with_sideinput/Makefile new file mode 100644 index 00000000..79c631e6 --- /dev/null +++ b/pkg/sideinput/examples/simple_source_with_sideinput/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/simple-source-example main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/simple-source-with-sideinput:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/simple-source-with-sideinput:${TAG}" --target sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/simple-source-with-sideinput:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/simple_source_with_sideinput/README.md b/pkg/sideinput/examples/simple_source_with_sideinput/README.md new file mode 100644 index 00000000..a7004e12 --- /dev/null +++ b/pkg/sideinput/examples/simple_source_with_sideinput/README.md @@ -0,0 +1,13 @@ +# Simple Source + +A simple example of a user-defined source. +The SimpleSource is a basic source implementation that listens for changes in a side +input file using a file watcher. When the specified side input file is created, +its content is read, stored in a global variable, and sent through a global channel. +This content is then consumed by the source and sent to the associated message channel. + +The source maintains an array of messages and implements the `Read()`, `Ack()`, and `Pending()` methods. + +The `Read(x)` function of the SimpleSource struct is responsible for reading data from the source (in this case, from the globalChan channel, which receives file content when a side input file is created) and sending this data to a specified message channel. +The `Ack()` method acknowledges the last batch of messages returned by `Read()`. +The `Pending()` method returns 0 to indicate that the simple source always has 0 pending messages. \ No newline at end of file diff --git a/pkg/sideinput/examples/simple_source_with_sideinput/go.mod b/pkg/sideinput/examples/simple_source_with_sideinput/go.mod new file mode 100644 index 00000000..8f7bbb07 --- /dev/null +++ b/pkg/sideinput/examples/simple_source_with_sideinput/go.mod @@ -0,0 +1,26 @@ +module simple_source_with_sideinput + +go 1.21 + +replace github.com/numaproj/numaflow-go => ../../../.. + +require ( + github.com/fsnotify/fsnotify v1.6.0 + github.com/numaproj/numaflow-go v0.7.0-rc2 + github.com/stretchr/testify v1.8.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/pkg/sideinput/examples/simple_source_with_sideinput/go.sum b/pkg/sideinput/examples/simple_source_with_sideinput/go.sum new file mode 100644 index 00000000..f3487675 --- /dev/null +++ b/pkg/sideinput/examples/simple_source_with_sideinput/go.sum @@ -0,0 +1,49 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/simple_source_with_sideinput/impl/simple_source_sideinput.go b/pkg/sideinput/examples/simple_source_with_sideinput/impl/simple_source_sideinput.go new file mode 100644 index 00000000..ffac5f17 --- /dev/null +++ b/pkg/sideinput/examples/simple_source_with_sideinput/impl/simple_source_sideinput.go @@ -0,0 +1,126 @@ +package impl + +import ( + "context" + "log" + "os" + "path" + "strconv" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + + "github.com/numaproj/numaflow-go/pkg/sideinput" + sourcesdk "github.com/numaproj/numaflow-go/pkg/sourcer" +) + +var sideInputMutex sync.Mutex + +// A global channel that will recev the data as soon as the file watcher sends it +var globalChan = make(chan string) +var contentValue string + +// SimpleSource is a simple source implementation. +type SimpleSource struct { + readIdx int64 + toAckSet map[int64]struct{} + lock *sync.Mutex +} + +func NewSimpleSource() *SimpleSource { + return &SimpleSource{ + readIdx: 0, + toAckSet: make(map[int64]struct{}), + lock: new(sync.Mutex), + } +} + +func (s *SimpleSource) Pending(_ context.Context) int64 { + // The simple source always returns zero to indicate there is no pending record. + return 0 +} + +func (s *SimpleSource) Read(_ context.Context, readRequest sourcesdk.ReadRequest, messageCh chan<- sourcesdk.Message) { + // Handle the timeout specification in the read request. + ctx, cancel := context.WithTimeout(context.Background(), readRequest.TimeOut()) + defer cancel() + + if len(s.toAckSet) > 0 { + return + } + + // Read the data from the source and send the data to the message channel. + for i := 0; uint64(i) < readRequest.Count(); i++ { + select { + case <-ctx.Done(): + // If the context is done, the read request is timed out. + return + case value := <-globalChan: + s.lock.Lock() + // Otherwise, we read the data from the source and send the data to the message channel. + offsetValue := serializeOffset(s.readIdx) + messageCh <- sourcesdk.NewMessage( + []byte(value), + sourcesdk.NewOffset(offsetValue, 0), + time.Now()) + // Mark the offset as to be acked, and increment the read index. + s.toAckSet[s.readIdx] = struct{}{} + s.readIdx++ + s.lock.Unlock() + + } + } + +} + +func (s *SimpleSource) Ack(_ context.Context, request sourcesdk.AckRequest) { + for _, offset := range request.Offsets() { + delete(s.toAckSet, deserializeOffset(offset.Value())) + } +} + +func (s *SimpleSource) Partitions(_ context.Context) []int32 { + return sourcesdk.DefaultPartitions() +} + +func serializeOffset(idx int64) []byte { + return []byte(strconv.FormatInt(idx, 10)) +} + +func deserializeOffset(offset []byte) int64 { + idx, _ := strconv.ParseInt(string(offset), 10, 64) + return idx +} + +func FileWatcher(watcher *fsnotify.Watcher, sideInputName string) { + log.Println("Watching for changes in side input file: ", sideinput.DirPath) + p := path.Join(sideinput.DirPath, sideInputName) + for { + select { + case event, ok := <-watcher.Events: + if !ok { + log.Println("watcher.Events channel closed") + return + } + if event.Op&fsnotify.Create == fsnotify.Create && event.Name == p { + log.Println("Side input file has been created:", event.Name) + b, err := os.ReadFile(p) + if err != nil { + log.Println("Failed to read side input file: ", err) + } + // Store the file content in the global variable and protect with mutex + sideInputMutex.Lock() + contentValue = string(b) + globalChan <- contentValue + sideInputMutex.Unlock() + } + case err, ok := <-watcher.Errors: + if !ok { + log.Println("watcher.Errors channel closed") + return + } + log.Println("error:", err) + } + } +} diff --git a/pkg/sideinput/examples/simple_source_with_sideinput/impl/simple_source_sideinput_test.go b/pkg/sideinput/examples/simple_source_with_sideinput/impl/simple_source_sideinput_test.go new file mode 100644 index 00000000..19f99aad --- /dev/null +++ b/pkg/sideinput/examples/simple_source_with_sideinput/impl/simple_source_sideinput_test.go @@ -0,0 +1,115 @@ +package impl + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/numaproj/numaflow-go/pkg/sourcer" +) + +// Notes: the unit test cases below demonstrate the basic contract between Read and Ack functions of a data source. +// These test cases should be applicable to all data sources, not just SimpleSource. + +type TestReadRequest struct { + count uint64 + timeout time.Duration +} + +func (rr TestReadRequest) Count() uint64 { + return rr.count +} + +func (rr TestReadRequest) TimeOut() time.Duration { + return rr.timeout +} + +type TestAckRequest struct { + offsets []sourcer.Offset +} + +func (ar TestAckRequest) Offsets() []sourcer.Offset { + return ar.offsets +} + +func TestNewSimpleSource(t *testing.T) { + underTest := NewSimpleSource() + // Prepare a channel to receive messages + messageCh := make(chan sourcer.Message, 20) + doneCh := make(chan struct{}) // to know when the go routine is done + + go func() { + underTest.Read(context.TODO(), TestReadRequest{ + count: 2, + timeout: time.Second, + }, messageCh) + close(doneCh) + }() + // We will send messages to GlobalChan + globalChan <- "test_data_1" + globalChan <- "test_data_2" + <-doneCh + assert.Equal(t, 2, len(messageCh)) + // Try reading 4 more messages + // Since the previous batch didn't get acked, the data source shouldn't allow us to read more messages + // We should get 0 messages, meaning the channel only holds the previous 2 messages + + go func() { + // Read 2 messages + underTest.Read(context.TODO(), TestReadRequest{ + count: 4, + timeout: time.Second, + }, messageCh) + }() + + assert.Equal(t, 2, len(messageCh)) + + // Ack the first batch + msg1 := <-messageCh + msg2 := <-messageCh + assert.Equal(t, "test_data_1", string(msg1.Value())) + assert.Equal(t, "test_data_2", string(msg2.Value())) + + underTest.Ack(context.TODO(), TestAckRequest{ + offsets: []sourcer.Offset{msg1.Offset(), msg2.Offset()}, + }) + doneCh2 := make(chan struct{}) // to know when the go routine is done + + // Reading 6 more messages by sending in the globalchan + go func() { + underTest.Read(context.TODO(), TestReadRequest{ + count: 6, + timeout: time.Second, + }, messageCh) + close(doneCh2) + }() + + // We will send messages to GlobalChan + globalChan <- "test_data_1" + globalChan <- "test_data_2" + globalChan <- "test_data_3" + globalChan <- "test_data_4" + globalChan <- "test_data_5" + globalChan <- "test_data_6" + <-doneCh2 + assert.Equal(t, 6, len(messageCh)) + + // Ack the second batch + msg3 := <-messageCh + msg4 := <-messageCh + msg5 := <-messageCh + msg6 := <-messageCh + msg7 := <-messageCh + msg8 := <-messageCh + assert.Equal(t, 0, len(messageCh)) + underTest.Ack(context.TODO(), TestAckRequest{ + offsets: []sourcer.Offset{ + msg3.Offset(), msg4.Offset(), + msg5.Offset(), msg6.Offset(), + msg7.Offset(), msg8.Offset(), + }, + }) + +} diff --git a/pkg/sideinput/examples/simple_source_with_sideinput/main.go b/pkg/sideinput/examples/simple_source_with_sideinput/main.go new file mode 100644 index 00000000..9f4c911f --- /dev/null +++ b/pkg/sideinput/examples/simple_source_with_sideinput/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + "log" + + "github.com/fsnotify/fsnotify" + + "github.com/numaproj/numaflow-go/pkg/sideinput" + "github.com/numaproj/numaflow-go/pkg/sourcer" + "simple_source_with_sideinput/impl" +) + +var sideInputName = "myticker" + +func main() { + // Create a new fsnotify watcher + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + // Add a path to the watcher + err = watcher.Add(sideinput.DirPath) + if err != nil { + log.Fatal(err) + } + go impl.FileWatcher(watcher, sideInputName) + + simpleSource := impl.NewSimpleSource() + err = sourcer.NewServer(simpleSource).Start(context.Background()) + if err != nil { + log.Panic("Failed to start source server : ", err) + } +} diff --git a/pkg/sideinput/examples/sink_sideinput/Dockerfile b/pkg/sideinput/examples/sink_sideinput/Dockerfile new file mode 100644 index 00000000..e67b592b --- /dev/null +++ b/pkg/sideinput/examples/sink_sideinput/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/redis-sink-with-sideinput /bin/redis-sink-with-sideinput +RUN chmod +x /bin/redis-sink-with-sideinput + +#################################################################################################### +# redis +#################################################################################################### +FROM scratch as sideinput +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/redis-sink-with-sideinput /bin/redis-sink-with-sideinput +ENTRYPOINT [ "/bin/redis-sink-with-sideinput" ] diff --git a/pkg/sideinput/examples/sink_sideinput/Makefile b/pkg/sideinput/examples/sink_sideinput/Makefile new file mode 100644 index 00000000..9bc0c3d5 --- /dev/null +++ b/pkg/sideinput/examples/sink_sideinput/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/redis-sink-with-sideinput main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/redis-sink-with-sideinput:${TAG}" --platform linux/amd64,linux/arm64 --target sideinput . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/redis-sink-with-sideinput:${TAG}" --target sideinput . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/redis-sink-with-sideinput:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sideinput/examples/sink_sideinput/README.md b/pkg/sideinput/examples/sink_sideinput/README.md new file mode 100644 index 00000000..c6a2e810 --- /dev/null +++ b/pkg/sideinput/examples/sink_sideinput/README.md @@ -0,0 +1,17 @@ +# Redis E2E Test Sink + +This Redis UDSink was specifically created for Numaflow end-to-end tests. +It interacts with a Redis instance and, upon receiving data, +it sets the content of the side input as the key in Redis, +and the number of occurrences of that key as its value. +The name of a hash is pipelineName:sinkName. + +# How It Works: +Uses the fsnotify package to watch for changes to the side input file. +Reads and stores the content of the side input file when created. +For each received message, the sink: +1. Uses the content of the side input as the Redis hash field (key) +2. Increments the value associated with that key (number of occurrences of that key) +3. The hash name in Redis is determined by the environment variables `NUMAFLOW_PIPELINE_NAME` and `NUMAFLOW_VERTEX_NAME` + + diff --git a/pkg/sideinput/examples/sink_sideinput/go.mod b/pkg/sideinput/examples/sink_sideinput/go.mod new file mode 100644 index 00000000..6521c8d7 --- /dev/null +++ b/pkg/sideinput/examples/sink_sideinput/go.mod @@ -0,0 +1,23 @@ +module sink_sideinput + +go 1.21 + +replace github.com/numaproj/numaflow-go => ../../../.. + +require ( + github.com/fsnotify/fsnotify v1.5.1 + github.com/go-redis/redis/v8 v8.11.4 + github.com/numaproj/numaflow-go v0.7.0-rc2 +) + +require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/sideinput/examples/sink_sideinput/go.sum b/pkg/sideinput/examples/sink_sideinput/go.sum new file mode 100644 index 00000000..48c5fc79 --- /dev/null +++ b/pkg/sideinput/examples/sink_sideinput/go.sum @@ -0,0 +1,119 @@ +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/sink_sideinput/main.go b/pkg/sideinput/examples/sink_sideinput/main.go new file mode 100644 index 00000000..3f5e1a0e --- /dev/null +++ b/pkg/sideinput/examples/sink_sideinput/main.go @@ -0,0 +1,108 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + "path" + "sync" + + "github.com/fsnotify/fsnotify" + "github.com/go-redis/redis/v8" + + "github.com/numaproj/numaflow-go/pkg/sideinput" + sinksdk "github.com/numaproj/numaflow-go/pkg/sinker" +) + +var sideInputName = "myticker" +var sideInputContent string +var sideInputMutex sync.Mutex + +// This redis UDSink is created for numaflow e2e tests. This handle function assumes that +// a redis instance listening on address redis:6379 has already be up and running. +func handle(ctx context.Context, datumStreamCh <-chan sinksdk.Datum) sinksdk.Responses { + client := redis.NewClient(&redis.Options{ + Addr: "redis:6379", + }) + + result := sinksdk.ResponsesBuilder() + for d := range datumStreamCh { + _ = d.EventTime() + _ = d.Watermark() + + // We use redis hashes to store messages. + // The name of a hash is pipelineName:sinkName. + // Each field of a hash is the content of a message and value of the field is the no. of occurrences of the message. + hkey := fmt.Sprintf("%s:%s", os.Getenv("NUMAFLOW_PIPELINE_NAME"), os.Getenv("NUMAFLOW_VERTEX_NAME")) + sideInputMutex.Lock() + content := sideInputContent + + sideInputMutex.Unlock() + err := client.HIncrBy(ctx, hkey, content, 1).Err() + if err != nil { + log.Println("Set Error - ", err) + } else { + log.Printf("Incremented by 1 the no. of occurrences of %s under hash key %s\n", content, hkey) + } + + id := d.ID() + result = result.Append(sinksdk.ResponseOK(id)) + } + return result +} + +func main() { + // Create a new fsnotify watcher + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Fatal(err) + } + defer watcher.Close() + + // Add a path to the watcher + err = watcher.Add(sideinput.DirPath) + if err != nil { + log.Fatal(err) + } + + // Start a goroutine to listen for events from the watcher + go fileWatcher(watcher, sideInputName) + err = sinksdk.NewServer(sinksdk.SinkerFunc(handle)).Start(context.Background()) + if err != nil { + log.Fatal(err) + + } +} + +func fileWatcher(watcher *fsnotify.Watcher, sideInputName string) { + log.Println("Watching for changes in side input file: ", sideinput.DirPath) + p := path.Join(sideinput.DirPath, sideInputName) + for { + select { + case event, ok := <-watcher.Events: + if !ok { + log.Println("watcher.Events channel closed") + return + } + if event.Op&fsnotify.Create == fsnotify.Create && event.Name == p { + log.Println("Side input file has been created:", event.Name) + b, err := os.ReadFile(p) + if err != nil { + log.Println("Failed to read side input file: ", err) + } + // Store the file content in the global variable and protect with mutex + sideInputMutex.Lock() + sideInputContent = string(b) + sideInputMutex.Unlock() + + } + case err, ok := <-watcher.Errors: + if !ok { + log.Println("watcher.Errors channel closed") + return + } + log.Println("error:", err) + } + } +} diff --git a/pkg/sinker/examples/fallback/go.mod b/pkg/sinker/examples/fallback/go.mod index 68c596f1..5eb20b2a 100644 --- a/pkg/sinker/examples/fallback/go.mod +++ b/pkg/sinker/examples/fallback/go.mod @@ -1,4 +1,4 @@ -module log_sink +module fallback go 1.20 diff --git a/pkg/sinker/examples/redis-sink/Dockerfile b/pkg/sinker/examples/redis-sink/Dockerfile new file mode 100644 index 00000000..3f6ffd4b --- /dev/null +++ b/pkg/sinker/examples/redis-sink/Dockerfile @@ -0,0 +1,20 @@ +#################################################################################################### +# base +#################################################################################################### +FROM alpine:3.12.3 as base +RUN apk update && apk upgrade && \ + apk add ca-certificates && \ + apk --no-cache add tzdata + +COPY dist/redis-sink /bin/redis-sink +RUN chmod +x /bin/redis-sink + +#################################################################################################### +# redis +#################################################################################################### +FROM scratch as redis +ARG ARCH +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=base /bin/redis-sink /bin/redis-sink +ENTRYPOINT [ "/bin/redis-sink" ] diff --git a/pkg/sinker/examples/redis-sink/Makefile b/pkg/sinker/examples/redis-sink/Makefile new file mode 100644 index 00000000..1d26a234 --- /dev/null +++ b/pkg/sinker/examples/redis-sink/Makefile @@ -0,0 +1,18 @@ +TAG ?= stable +PUSH ?= false + +.PHONY: build +build: + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o ./dist/redis-sink main.go + +.PHONY: image-push +image-push: build + docker buildx build -t "quay.io/numaio/numaflow-go/redis-sink:${TAG}" --platform linux/amd64,linux/arm64 --target redis . --push + +.PHONY: image +image: build + docker build -t "quay.io/numaio/numaflow-go/redis-sink:${TAG}" --target redis . + @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/redis-sink:${TAG}"; fi + +clean: + -rm -rf ./dist diff --git a/pkg/sinker/examples/redis-sink/README.md b/pkg/sinker/examples/redis-sink/README.md new file mode 100644 index 00000000..4b41a662 --- /dev/null +++ b/pkg/sinker/examples/redis-sink/README.md @@ -0,0 +1,9 @@ +# Redis E2E Test Sink +A User Defined Sink using redis hashes to store messages. +The name of a hash is pipelineName:sinkName. + +For each message received, the sink will store the message in a hash with the key being the payload of the message +and the value being the no. of occurrences of that payload so far. + +This sink is used by Numaflow E2E testing. + diff --git a/pkg/sinker/examples/redis-sink/go.mod b/pkg/sinker/examples/redis-sink/go.mod new file mode 100644 index 00000000..72696c28 --- /dev/null +++ b/pkg/sinker/examples/redis-sink/go.mod @@ -0,0 +1,23 @@ +module redis-e2e-test-sink + +go 1.18 + +replace github.com/numaproj/numaflow-go => ../../../.. + +require ( + github.com/go-redis/redis/v8 v8.11.4 + github.com/numaproj/numaflow-go v0.7.0-rc2 +) + +require ( + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/fsnotify/fsnotify v1.5.1 // indirect + github.com/golang/protobuf v1.5.3 // indirect + golang.org/x/net v0.9.0 // indirect + golang.org/x/sys v0.7.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + google.golang.org/grpc v1.57.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect +) diff --git a/pkg/sinker/examples/redis-sink/go.sum b/pkg/sinker/examples/redis-sink/go.sum new file mode 100644 index 00000000..31d62b2d --- /dev/null +++ b/pkg/sinker/examples/redis-sink/go.sum @@ -0,0 +1,116 @@ +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/go-redis/redis/v8 v8.11.4 h1:kHoYkfZP6+pe04aFTnhDH6GDROa5yJdHJVNxV3F46Tg= +github.com/go-redis/redis/v8 v8.11.4/go.mod h1:2Z2wHZXdQpCDXEGzqMockDpNyYvi2l4Pxt6RJr792+w= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/pkg/sinker/examples/redis-sink/main.go b/pkg/sinker/examples/redis-sink/main.go new file mode 100644 index 00000000..94603190 --- /dev/null +++ b/pkg/sinker/examples/redis-sink/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/go-redis/redis/v8" + + sinksdk "github.com/numaproj/numaflow-go/pkg/sinker" +) + +type redisTestSink struct{} + +// Sink This redis UDSink is created for numaflow e2e tests. This handle function assumes that +// a redis instance listening on address redis:6379 has already be up and running. +func (rds *redisTestSink) Sink(ctx context.Context, datumStreamCh <-chan sinksdk.Datum) sinksdk.Responses { + client := redis.NewClient(&redis.Options{ + Addr: "redis:6379", + }) + + result := sinksdk.ResponsesBuilder() + for d := range datumStreamCh { + _ = d.EventTime() + _ = d.Watermark() + + // We use redis hashes to store messages. + // The name of a hash is pipelineName:sinkName. + // Each field of a hash is the content of a message and value of the field is the no. of occurrences of the message. + hkey := fmt.Sprintf("%s:%s", os.Getenv("NUMAFLOW_PIPELINE_NAME"), os.Getenv("NUMAFLOW_VERTEX_NAME")) + err := client.HIncrBy(ctx, hkey, string(d.Value()), 1).Err() + if err != nil { + log.Println("Set Error - ", err) + } else { + log.Printf("Incremented by 1 the no. of occurrences of %s under hash key %s\n", string(d.Value()), hkey) + } + + id := d.ID() + result = result.Append(sinksdk.ResponseOK(id)) + } + return result +} + +func main() { + err := sinksdk.NewServer(&redisTestSink{}).Start(context.Background()) + if err != nil { + log.Panic("Failed to start sink function server: ", err) + } +}