diff --git a/go.mod b/go.mod index 423fb94f..5d75e67a 100644 --- a/go.mod +++ b/go.mod @@ -1,26 +1,25 @@ module github.com/numaproj/numaflow-go -go 1.20 +go 1.21 require ( github.com/golang/mock v1.6.0 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.9.0 go.uber.org/atomic v1.11.0 - golang.org/x/net v0.9.0 - golang.org/x/sync v0.1.0 - google.golang.org/grpc v1.57.0 - google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 - google.golang.org/protobuf v1.31.0 + golang.org/x/net v0.29.0 + golang.org/x/sync v0.8.0 + google.golang.org/grpc v1.66.0 + google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.4.0 + google.golang.org/protobuf v1.34.2 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index c20884dc..1a5ca3a6 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,10 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -18,13 +14,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -34,44 +25,40 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA= -google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.4.0 h1:9SxA29VM43MF5Z9dQu694wmY5t8E/Gxr7s+RSxiIDmc= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.4.0/go.mod h1:yZOK5zhQMiALmuweVdIVoQPa6eIJyXn2B9g5dJDhqX4= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/apis/proto/batchmap/v1/batchmap.pb.go b/pkg/apis/proto/batchmap/v1/batchmap.pb.go deleted file mode 100644 index 1d34241c..00000000 --- a/pkg/apis/proto/batchmap/v1/batchmap.pb.go +++ /dev/null @@ -1,456 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.28.1 -// protoc v3.21.12 -// source: pkg/apis/proto/batchmap/v1/batchmap.proto - -package v1 - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - emptypb "google.golang.org/protobuf/types/known/emptypb" - timestamppb "google.golang.org/protobuf/types/known/timestamppb" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// * -// BatchMapRequest represents a request element. -type BatchMapRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` - Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` - Headers map[string]string `protobuf:"bytes,5,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - // This ID is used uniquely identify a map request - Id string `protobuf:"bytes,6,opt,name=id,proto3" json:"id,omitempty"` -} - -func (x *BatchMapRequest) Reset() { - *x = BatchMapRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchMapRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchMapRequest) ProtoMessage() {} - -func (x *BatchMapRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchMapRequest.ProtoReflect.Descriptor instead. -func (*BatchMapRequest) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDescGZIP(), []int{0} -} - -func (x *BatchMapRequest) GetKeys() []string { - if x != nil { - return x.Keys - } - return nil -} - -func (x *BatchMapRequest) GetValue() []byte { - if x != nil { - return x.Value - } - return nil -} - -func (x *BatchMapRequest) GetEventTime() *timestamppb.Timestamp { - if x != nil { - return x.EventTime - } - return nil -} - -func (x *BatchMapRequest) GetWatermark() *timestamppb.Timestamp { - if x != nil { - return x.Watermark - } - return nil -} - -func (x *BatchMapRequest) GetHeaders() map[string]string { - if x != nil { - return x.Headers - } - return nil -} - -func (x *BatchMapRequest) GetId() string { - if x != nil { - return x.Id - } - return "" -} - -// * -// BatchMapResponse represents a response element. -type BatchMapResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Results []*BatchMapResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` - // This ID is used to refer the responses to the request it corresponds to. - Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` -} - -func (x *BatchMapResponse) Reset() { - *x = BatchMapResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchMapResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchMapResponse) ProtoMessage() {} - -func (x *BatchMapResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchMapResponse.ProtoReflect.Descriptor instead. -func (*BatchMapResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDescGZIP(), []int{1} -} - -func (x *BatchMapResponse) GetResults() []*BatchMapResponse_Result { - if x != nil { - return x.Results - } - return nil -} - -func (x *BatchMapResponse) GetId() string { - if x != nil { - return x.Id - } - return "" -} - -// * -// ReadyResponse is the health check result. -type ReadyResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` -} - -func (x *ReadyResponse) Reset() { - *x = ReadyResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ReadyResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ReadyResponse) ProtoMessage() {} - -func (x *ReadyResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ReadyResponse.ProtoReflect.Descriptor instead. -func (*ReadyResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDescGZIP(), []int{2} -} - -func (x *ReadyResponse) GetReady() bool { - if x != nil { - return x.Ready - } - return false -} - -type BatchMapResponse_Result struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` -} - -func (x *BatchMapResponse_Result) Reset() { - *x = BatchMapResponse_Result{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *BatchMapResponse_Result) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*BatchMapResponse_Result) ProtoMessage() {} - -func (x *BatchMapResponse_Result) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use BatchMapResponse_Result.ProtoReflect.Descriptor instead. -func (*BatchMapResponse_Result) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDescGZIP(), []int{1, 0} -} - -func (x *BatchMapResponse_Result) GetKeys() []string { - if x != nil { - return x.Keys - } - return nil -} - -func (x *BatchMapResponse_Result) GetValue() []byte { - if x != nil { - return x.Value - } - return nil -} - -func (x *BatchMapResponse_Result) GetTags() []string { - if x != nil { - return x.Tags - } - return nil -} - -var File_pkg_apis_proto_batchmap_v1_batchmap_proto protoreflect.FileDescriptor - -var file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDesc = []byte{ - 0x0a, 0x29, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2f, 0x62, 0x61, 0x74, 0x63, 0x68, 0x6d, 0x61, 0x70, 0x2f, 0x76, 0x31, 0x2f, 0x62, 0x61, 0x74, - 0x63, 0x68, 0x6d, 0x61, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0b, 0x62, 0x61, 0x74, - 0x63, 0x68, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc1, 0x02, 0x0a, 0x0f, 0x42, 0x61, 0x74, 0x63, 0x68, - 0x4d, 0x61, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, - 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, - 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, - 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, - 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, - 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x12, 0x43, 0x0a, 0x07, 0x68, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x62, 0x61, 0x74, - 0x63, 0x68, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x61, - 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x1a, 0x3a, - 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, - 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xaa, 0x01, 0x0a, 0x10, 0x42, - 0x61, 0x74, 0x63, 0x68, 0x4d, 0x61, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x3e, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x24, 0x2e, 0x62, 0x61, 0x74, 0x63, 0x68, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x42, - 0x61, 0x74, 0x63, 0x68, 0x4d, 0x61, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, - 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x1a, - 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, - 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0x98, - 0x01, 0x0a, 0x08, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x61, 0x70, 0x12, 0x3d, 0x0a, 0x07, 0x49, - 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1a, - 0x2e, 0x62, 0x61, 0x74, 0x63, 0x68, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, - 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x0a, 0x42, 0x61, - 0x74, 0x63, 0x68, 0x4d, 0x61, 0x70, 0x46, 0x6e, 0x12, 0x1c, 0x2e, 0x62, 0x61, 0x74, 0x63, 0x68, - 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x61, 0x70, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x62, 0x61, 0x74, 0x63, 0x68, 0x6d, 0x61, - 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x42, 0x61, 0x74, 0x63, 0x68, 0x4d, 0x61, 0x70, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x42, 0x3c, 0x5a, 0x3a, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, - 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, - 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x62, 0x61, 0x74, 0x63, - 0x68, 0x6d, 0x61, 0x70, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDescOnce sync.Once - file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDescData = file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDesc -) - -func file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDescGZIP() []byte { - file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDescOnce.Do(func() { - file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDescData) - }) - return file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDescData -} - -var file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes = make([]protoimpl.MessageInfo, 5) -var file_pkg_apis_proto_batchmap_v1_batchmap_proto_goTypes = []interface{}{ - (*BatchMapRequest)(nil), // 0: batchmap.v1.BatchMapRequest - (*BatchMapResponse)(nil), // 1: batchmap.v1.BatchMapResponse - (*ReadyResponse)(nil), // 2: batchmap.v1.ReadyResponse - nil, // 3: batchmap.v1.BatchMapRequest.HeadersEntry - (*BatchMapResponse_Result)(nil), // 4: batchmap.v1.BatchMapResponse.Result - (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 6: google.protobuf.Empty -} -var file_pkg_apis_proto_batchmap_v1_batchmap_proto_depIdxs = []int32{ - 5, // 0: batchmap.v1.BatchMapRequest.event_time:type_name -> google.protobuf.Timestamp - 5, // 1: batchmap.v1.BatchMapRequest.watermark:type_name -> google.protobuf.Timestamp - 3, // 2: batchmap.v1.BatchMapRequest.headers:type_name -> batchmap.v1.BatchMapRequest.HeadersEntry - 4, // 3: batchmap.v1.BatchMapResponse.results:type_name -> batchmap.v1.BatchMapResponse.Result - 6, // 4: batchmap.v1.BatchMap.IsReady:input_type -> google.protobuf.Empty - 0, // 5: batchmap.v1.BatchMap.BatchMapFn:input_type -> batchmap.v1.BatchMapRequest - 2, // 6: batchmap.v1.BatchMap.IsReady:output_type -> batchmap.v1.ReadyResponse - 1, // 7: batchmap.v1.BatchMap.BatchMapFn:output_type -> batchmap.v1.BatchMapResponse - 6, // [6:8] is the sub-list for method output_type - 4, // [4:6] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name -} - -func init() { file_pkg_apis_proto_batchmap_v1_batchmap_proto_init() } -func file_pkg_apis_proto_batchmap_v1_batchmap_proto_init() { - if File_pkg_apis_proto_batchmap_v1_batchmap_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchMapRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchMapResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReadyResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*BatchMapResponse_Result); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDesc, - NumEnums: 0, - NumMessages: 5, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_pkg_apis_proto_batchmap_v1_batchmap_proto_goTypes, - DependencyIndexes: file_pkg_apis_proto_batchmap_v1_batchmap_proto_depIdxs, - MessageInfos: file_pkg_apis_proto_batchmap_v1_batchmap_proto_msgTypes, - }.Build() - File_pkg_apis_proto_batchmap_v1_batchmap_proto = out.File - file_pkg_apis_proto_batchmap_v1_batchmap_proto_rawDesc = nil - file_pkg_apis_proto_batchmap_v1_batchmap_proto_goTypes = nil - file_pkg_apis_proto_batchmap_v1_batchmap_proto_depIdxs = nil -} diff --git a/pkg/apis/proto/batchmap/v1/batchmap.proto b/pkg/apis/proto/batchmap/v1/batchmap.proto deleted file mode 100644 index f7f4d10e..00000000 --- a/pkg/apis/proto/batchmap/v1/batchmap.proto +++ /dev/null @@ -1,52 +0,0 @@ -syntax = "proto3"; - -option go_package = "github.com/numaproj/numaflow-go/pkg/apis/proto/batchmap/v1"; - -import "google/protobuf/empty.proto"; -import "google/protobuf/timestamp.proto"; - -package batchmap.v1; - -service BatchMap { - // IsReady is the heartbeat endpoint for gRPC. - rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); - - // BatchMapFn is a bi-directional streaming rpc which applies a - // map function on each BatchMapRequest element of the stream and then streams - // back BatchMapResponse elements. - rpc BatchMapFn(stream BatchMapRequest) returns (stream BatchMapResponse); -} - -/** - * BatchMapRequest represents a request element. - */ -message BatchMapRequest { - repeated string keys = 1; - bytes value = 2; - google.protobuf.Timestamp event_time = 3; - google.protobuf.Timestamp watermark = 4; - map headers = 5; - // This ID is used uniquely identify a map request - string id = 6; -} - -/** - * BatchMapResponse represents a response element. - */ -message BatchMapResponse { - message Result { - repeated string keys = 1; - bytes value = 2; - repeated string tags = 3; - } - repeated Result results = 1; - // This ID is used to refer the responses to the request it corresponds to. - string id = 2; -} - -/** - * ReadyResponse is the health check result. - */ -message ReadyResponse { - bool ready = 1; -} \ No newline at end of file diff --git a/pkg/apis/proto/batchmap/v1/batchmap_grpc.pb.go b/pkg/apis/proto/batchmap/v1/batchmap_grpc.pb.go deleted file mode 100644 index 11abd087..00000000 --- a/pkg/apis/proto/batchmap/v1/batchmap_grpc.pb.go +++ /dev/null @@ -1,183 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. -// versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v3.21.12 -// source: pkg/apis/proto/batchmap/v1/batchmap.proto - -package v1 - -import ( - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - emptypb "google.golang.org/protobuf/types/known/emptypb" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 - -// BatchMapClient is the client API for BatchMap service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type BatchMapClient interface { - // IsReady is the heartbeat endpoint for gRPC. - IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) - // BatchMapFn is a bi-directional streaming rpc which applies a - // map function on each BatchMapRequest element of the stream and then streams - // back BatchMapResponse elements. - BatchMapFn(ctx context.Context, opts ...grpc.CallOption) (BatchMap_BatchMapFnClient, error) -} - -type batchMapClient struct { - cc grpc.ClientConnInterface -} - -func NewBatchMapClient(cc grpc.ClientConnInterface) BatchMapClient { - return &batchMapClient{cc} -} - -func (c *batchMapClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { - out := new(ReadyResponse) - err := c.cc.Invoke(ctx, "/batchmap.v1.BatchMap/IsReady", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *batchMapClient) BatchMapFn(ctx context.Context, opts ...grpc.CallOption) (BatchMap_BatchMapFnClient, error) { - stream, err := c.cc.NewStream(ctx, &BatchMap_ServiceDesc.Streams[0], "/batchmap.v1.BatchMap/BatchMapFn", opts...) - if err != nil { - return nil, err - } - x := &batchMapBatchMapFnClient{stream} - return x, nil -} - -type BatchMap_BatchMapFnClient interface { - Send(*BatchMapRequest) error - Recv() (*BatchMapResponse, error) - grpc.ClientStream -} - -type batchMapBatchMapFnClient struct { - grpc.ClientStream -} - -func (x *batchMapBatchMapFnClient) Send(m *BatchMapRequest) error { - return x.ClientStream.SendMsg(m) -} - -func (x *batchMapBatchMapFnClient) Recv() (*BatchMapResponse, error) { - m := new(BatchMapResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// BatchMapServer is the server API for BatchMap service. -// All implementations must embed UnimplementedBatchMapServer -// for forward compatibility -type BatchMapServer interface { - // IsReady is the heartbeat endpoint for gRPC. - IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) - // BatchMapFn is a bi-directional streaming rpc which applies a - // map function on each BatchMapRequest element of the stream and then streams - // back BatchMapResponse elements. - BatchMapFn(BatchMap_BatchMapFnServer) error - mustEmbedUnimplementedBatchMapServer() -} - -// UnimplementedBatchMapServer must be embedded to have forward compatible implementations. -type UnimplementedBatchMapServer struct { -} - -func (UnimplementedBatchMapServer) IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method IsReady not implemented") -} -func (UnimplementedBatchMapServer) BatchMapFn(BatchMap_BatchMapFnServer) error { - return status.Errorf(codes.Unimplemented, "method BatchMapFn not implemented") -} -func (UnimplementedBatchMapServer) mustEmbedUnimplementedBatchMapServer() {} - -// UnsafeBatchMapServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to BatchMapServer will -// result in compilation errors. -type UnsafeBatchMapServer interface { - mustEmbedUnimplementedBatchMapServer() -} - -func RegisterBatchMapServer(s grpc.ServiceRegistrar, srv BatchMapServer) { - s.RegisterService(&BatchMap_ServiceDesc, srv) -} - -func _BatchMap_IsReady_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(emptypb.Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(BatchMapServer).IsReady(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/batchmap.v1.BatchMap/IsReady", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(BatchMapServer).IsReady(ctx, req.(*emptypb.Empty)) - } - return interceptor(ctx, in, info, handler) -} - -func _BatchMap_BatchMapFn_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(BatchMapServer).BatchMapFn(&batchMapBatchMapFnServer{stream}) -} - -type BatchMap_BatchMapFnServer interface { - Send(*BatchMapResponse) error - Recv() (*BatchMapRequest, error) - grpc.ServerStream -} - -type batchMapBatchMapFnServer struct { - grpc.ServerStream -} - -func (x *batchMapBatchMapFnServer) Send(m *BatchMapResponse) error { - return x.ServerStream.SendMsg(m) -} - -func (x *batchMapBatchMapFnServer) Recv() (*BatchMapRequest, error) { - m := new(BatchMapRequest) - if err := x.ServerStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// BatchMap_ServiceDesc is the grpc.ServiceDesc for BatchMap service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var BatchMap_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "batchmap.v1.BatchMap", - HandlerType: (*BatchMapServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "IsReady", - Handler: _BatchMap_IsReady_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "BatchMapFn", - Handler: _BatchMap_BatchMapFn_Handler, - ServerStreams: true, - ClientStreams: true, - }, - }, - Metadata: "pkg/apis/proto/batchmap/v1/batchmap.proto", -} diff --git a/pkg/apis/proto/batchmap/v1/batchmapmock/batchmapmock.go b/pkg/apis/proto/batchmap/v1/batchmapmock/batchmapmock.go deleted file mode 100644 index 43f0315b..00000000 --- a/pkg/apis/proto/batchmap/v1/batchmapmock/batchmapmock.go +++ /dev/null @@ -1,216 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/numaproj/numaflow-go/pkg/apis/proto/batchmap/v1 (interfaces: BatchMapClient,BatchMap_BatchMapFnClient) - -// Package batchmapmock is a generated GoMock package. -package batchmapmock - -import ( - context "context" - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/batchmap/v1" - grpc "google.golang.org/grpc" - metadata "google.golang.org/grpc/metadata" - emptypb "google.golang.org/protobuf/types/known/emptypb" -) - -// MockBatchMapClient is a mock of BatchMapClient interface. -type MockBatchMapClient struct { - ctrl *gomock.Controller - recorder *MockBatchMapClientMockRecorder -} - -// MockBatchMapClientMockRecorder is the mock recorder for MockBatchMapClient. -type MockBatchMapClientMockRecorder struct { - mock *MockBatchMapClient -} - -// NewMockBatchMapClient creates a new mock instance. -func NewMockBatchMapClient(ctrl *gomock.Controller) *MockBatchMapClient { - mock := &MockBatchMapClient{ctrl: ctrl} - mock.recorder = &MockBatchMapClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockBatchMapClient) EXPECT() *MockBatchMapClientMockRecorder { - return m.recorder -} - -// BatchMapFn mocks base method. -func (m *MockBatchMapClient) BatchMapFn(arg0 context.Context, arg1 ...grpc.CallOption) (v1.BatchMap_BatchMapFnClient, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0} - for _, a := range arg1 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "BatchMapFn", varargs...) - ret0, _ := ret[0].(v1.BatchMap_BatchMapFnClient) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// BatchMapFn indicates an expected call of BatchMapFn. -func (mr *MockBatchMapClientMockRecorder) BatchMapFn(arg0 interface{}, arg1 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0}, arg1...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BatchMapFn", reflect.TypeOf((*MockBatchMapClient)(nil).BatchMapFn), varargs...) -} - -// IsReady mocks base method. -func (m *MockBatchMapClient) IsReady(arg0 context.Context, arg1 *emptypb.Empty, arg2 ...grpc.CallOption) (*v1.ReadyResponse, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "IsReady", varargs...) - ret0, _ := ret[0].(*v1.ReadyResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// IsReady indicates an expected call of IsReady. -func (mr *MockBatchMapClientMockRecorder) IsReady(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsReady", reflect.TypeOf((*MockBatchMapClient)(nil).IsReady), varargs...) -} - -// MockBatchMap_BatchMapFnClient is a mock of BatchMap_BatchMapFnClient interface. -type MockBatchMap_BatchMapFnClient struct { - ctrl *gomock.Controller - recorder *MockBatchMap_BatchMapFnClientMockRecorder -} - -// MockBatchMap_BatchMapFnClientMockRecorder is the mock recorder for MockBatchMap_BatchMapFnClient. -type MockBatchMap_BatchMapFnClientMockRecorder struct { - mock *MockBatchMap_BatchMapFnClient -} - -// NewMockBatchMap_BatchMapFnClient creates a new mock instance. -func NewMockBatchMap_BatchMapFnClient(ctrl *gomock.Controller) *MockBatchMap_BatchMapFnClient { - mock := &MockBatchMap_BatchMapFnClient{ctrl: ctrl} - mock.recorder = &MockBatchMap_BatchMapFnClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockBatchMap_BatchMapFnClient) EXPECT() *MockBatchMap_BatchMapFnClientMockRecorder { - return m.recorder -} - -// CloseSend mocks base method. -func (m *MockBatchMap_BatchMapFnClient) CloseSend() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CloseSend") - ret0, _ := ret[0].(error) - return ret0 -} - -// CloseSend indicates an expected call of CloseSend. -func (mr *MockBatchMap_BatchMapFnClientMockRecorder) CloseSend() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseSend", reflect.TypeOf((*MockBatchMap_BatchMapFnClient)(nil).CloseSend)) -} - -// Context mocks base method. -func (m *MockBatchMap_BatchMapFnClient) Context() context.Context { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Context") - ret0, _ := ret[0].(context.Context) - return ret0 -} - -// Context indicates an expected call of Context. -func (mr *MockBatchMap_BatchMapFnClientMockRecorder) Context() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Context", reflect.TypeOf((*MockBatchMap_BatchMapFnClient)(nil).Context)) -} - -// Header mocks base method. -func (m *MockBatchMap_BatchMapFnClient) Header() (metadata.MD, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Header") - ret0, _ := ret[0].(metadata.MD) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Header indicates an expected call of Header. -func (mr *MockBatchMap_BatchMapFnClientMockRecorder) Header() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Header", reflect.TypeOf((*MockBatchMap_BatchMapFnClient)(nil).Header)) -} - -// Recv mocks base method. -func (m *MockBatchMap_BatchMapFnClient) Recv() (*v1.BatchMapResponse, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Recv") - ret0, _ := ret[0].(*v1.BatchMapResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Recv indicates an expected call of Recv. -func (mr *MockBatchMap_BatchMapFnClientMockRecorder) Recv() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Recv", reflect.TypeOf((*MockBatchMap_BatchMapFnClient)(nil).Recv)) -} - -// RecvMsg mocks base method. -func (m *MockBatchMap_BatchMapFnClient) RecvMsg(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RecvMsg", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// RecvMsg indicates an expected call of RecvMsg. -func (mr *MockBatchMap_BatchMapFnClientMockRecorder) RecvMsg(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockBatchMap_BatchMapFnClient)(nil).RecvMsg), arg0) -} - -// Send mocks base method. -func (m *MockBatchMap_BatchMapFnClient) Send(arg0 *v1.BatchMapRequest) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Send", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Send indicates an expected call of Send. -func (mr *MockBatchMap_BatchMapFnClientMockRecorder) Send(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockBatchMap_BatchMapFnClient)(nil).Send), arg0) -} - -// SendMsg mocks base method. -func (m *MockBatchMap_BatchMapFnClient) SendMsg(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMsg", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendMsg indicates an expected call of SendMsg. -func (mr *MockBatchMap_BatchMapFnClientMockRecorder) SendMsg(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockBatchMap_BatchMapFnClient)(nil).SendMsg), arg0) -} - -// Trailer mocks base method. -func (m *MockBatchMap_BatchMapFnClient) Trailer() metadata.MD { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Trailer") - ret0, _ := ret[0].(metadata.MD) - return ret0 -} - -// Trailer indicates an expected call of Trailer. -func (mr *MockBatchMap_BatchMapFnClientMockRecorder) Trailer() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Trailer", reflect.TypeOf((*MockBatchMap_BatchMapFnClient)(nil).Trailer)) -} diff --git a/pkg/apis/proto/batchmap/v1/mockgen.go b/pkg/apis/proto/batchmap/v1/mockgen.go deleted file mode 100644 index 04982aeb..00000000 --- a/pkg/apis/proto/batchmap/v1/mockgen.go +++ /dev/null @@ -1,3 +0,0 @@ -package v1 - -//go:generate mockgen -destination batchmapmock/batchmapmock.go -package batchmapmock github.com/numaproj/numaflow-go/pkg/apis/proto/batchmap/v1 BatchMapClient,BatchMap_BatchMapFnClient diff --git a/pkg/apis/proto/map/v1/map.pb.go b/pkg/apis/proto/map/v1/map.pb.go index 58f978de..752e20c1 100644 --- a/pkg/apis/proto/map/v1/map.pb.go +++ b/pkg/apis/proto/map/v1/map.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.34.2 // protoc v4.25.1 // source: pkg/apis/proto/map/v1/map.proto @@ -29,11 +29,11 @@ type MapRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` - Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` - Headers map[string]string `protobuf:"bytes,5,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Request *MapRequest_Request `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` + // This ID is used to uniquely identify a map request + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + Handshake *Handshake `protobuf:"bytes,3,opt,name=handshake,proto3,oneof" json:"handshake,omitempty"` + Status *TransmissionStatus `protobuf:"bytes,4,opt,name=status,proto3,oneof" json:"status,omitempty"` } func (x *MapRequest) Reset() { @@ -68,39 +68,129 @@ func (*MapRequest) Descriptor() ([]byte, []int) { return file_pkg_apis_proto_map_v1_map_proto_rawDescGZIP(), []int{0} } -func (x *MapRequest) GetKeys() []string { +func (x *MapRequest) GetRequest() *MapRequest_Request { if x != nil { - return x.Keys + return x.Request } return nil } -func (x *MapRequest) GetValue() []byte { +func (x *MapRequest) GetId() string { if x != nil { - return x.Value + return x.Id } - return nil + return "" } -func (x *MapRequest) GetEventTime() *timestamppb.Timestamp { +func (x *MapRequest) GetHandshake() *Handshake { if x != nil { - return x.EventTime + return x.Handshake } return nil } -func (x *MapRequest) GetWatermark() *timestamppb.Timestamp { +func (x *MapRequest) GetStatus() *TransmissionStatus { if x != nil { - return x.Watermark + return x.Status } return nil } -func (x *MapRequest) GetHeaders() map[string]string { +// Handshake message between client and server to indicate the start of transmission. +type Handshake struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Required field indicating the start of transmission. + Sot bool `protobuf:"varint,1,opt,name=sot,proto3" json:"sot,omitempty"` +} + +func (x *Handshake) Reset() { + *x = Handshake{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Handshake) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Handshake) ProtoMessage() {} + +func (x *Handshake) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Handshake.ProtoReflect.Descriptor instead. +func (*Handshake) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_map_v1_map_proto_rawDescGZIP(), []int{1} +} + +func (x *Handshake) GetSot() bool { if x != nil { - return x.Headers + return x.Sot } - return nil + return false +} + +// Status message to indicate the status of the message. +type TransmissionStatus struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Eot bool `protobuf:"varint,1,opt,name=eot,proto3" json:"eot,omitempty"` +} + +func (x *TransmissionStatus) Reset() { + *x = TransmissionStatus{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TransmissionStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TransmissionStatus) ProtoMessage() {} + +func (x *TransmissionStatus) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TransmissionStatus.ProtoReflect.Descriptor instead. +func (*TransmissionStatus) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_map_v1_map_proto_rawDescGZIP(), []int{2} +} + +func (x *TransmissionStatus) GetEot() bool { + if x != nil { + return x.Eot + } + return false } // * @@ -111,12 +201,16 @@ type MapResponse struct { unknownFields protoimpl.UnknownFields Results []*MapResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + // This ID is used to refer the responses to the request it corresponds to. + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + Handshake *Handshake `protobuf:"bytes,3,opt,name=handshake,proto3,oneof" json:"handshake,omitempty"` + Status *TransmissionStatus `protobuf:"bytes,4,opt,name=status,proto3,oneof" json:"status,omitempty"` } func (x *MapResponse) Reset() { *x = MapResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[1] + mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -129,7 +223,7 @@ func (x *MapResponse) String() string { func (*MapResponse) ProtoMessage() {} func (x *MapResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[1] + mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -142,7 +236,7 @@ func (x *MapResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use MapResponse.ProtoReflect.Descriptor instead. func (*MapResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_map_v1_map_proto_rawDescGZIP(), []int{1} + return file_pkg_apis_proto_map_v1_map_proto_rawDescGZIP(), []int{3} } func (x *MapResponse) GetResults() []*MapResponse_Result { @@ -152,6 +246,27 @@ func (x *MapResponse) GetResults() []*MapResponse_Result { return nil } +func (x *MapResponse) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *MapResponse) GetHandshake() *Handshake { + if x != nil { + return x.Handshake + } + return nil +} + +func (x *MapResponse) GetStatus() *TransmissionStatus { + if x != nil { + return x.Status + } + return nil +} + // * // ReadyResponse is the health check result. type ReadyResponse struct { @@ -165,7 +280,7 @@ type ReadyResponse struct { func (x *ReadyResponse) Reset() { *x = ReadyResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[2] + mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -178,7 +293,7 @@ func (x *ReadyResponse) String() string { func (*ReadyResponse) ProtoMessage() {} func (x *ReadyResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[2] + mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -191,7 +306,7 @@ func (x *ReadyResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadyResponse.ProtoReflect.Descriptor instead. func (*ReadyResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_map_v1_map_proto_rawDescGZIP(), []int{2} + return file_pkg_apis_proto_map_v1_map_proto_rawDescGZIP(), []int{4} } func (x *ReadyResponse) GetReady() bool { @@ -201,6 +316,85 @@ func (x *ReadyResponse) GetReady() bool { return false } +type MapRequest_Request struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` + Headers map[string]string `protobuf:"bytes,5,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *MapRequest_Request) Reset() { + *x = MapRequest_Request{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MapRequest_Request) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MapRequest_Request) ProtoMessage() {} + +func (x *MapRequest_Request) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MapRequest_Request.ProtoReflect.Descriptor instead. +func (*MapRequest_Request) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_map_v1_map_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *MapRequest_Request) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + +func (x *MapRequest_Request) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *MapRequest_Request) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + +func (x *MapRequest_Request) GetWatermark() *timestamppb.Timestamp { + if x != nil { + return x.Watermark + } + return nil +} + +func (x *MapRequest_Request) GetHeaders() map[string]string { + if x != nil { + return x.Headers + } + return nil +} + type MapResponse_Result struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -214,7 +408,7 @@ type MapResponse_Result struct { func (x *MapResponse_Result) Reset() { *x = MapResponse_Result{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[4] + mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -227,7 +421,7 @@ func (x *MapResponse_Result) String() string { func (*MapResponse_Result) ProtoMessage() {} func (x *MapResponse_Result) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[4] + mi := &file_pkg_apis_proto_map_v1_map_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -240,7 +434,7 @@ func (x *MapResponse_Result) ProtoReflect() protoreflect.Message { // Deprecated: Use MapResponse_Result.ProtoReflect.Descriptor instead. func (*MapResponse_Result) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_map_v1_map_proto_rawDescGZIP(), []int{1, 0} + return file_pkg_apis_proto_map_v1_map_proto_rawDescGZIP(), []int{3, 0} } func (x *MapResponse_Result) GetKeys() []string { @@ -273,48 +467,76 @@ var file_pkg_apis_proto_map_v1_map_proto_rawDesc = []byte{ 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xa2, 0x02, 0x0a, 0x0a, 0x4d, 0x61, 0x70, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, - 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, - 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x12, 0x39, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, - 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, - 0x4d, 0x61, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, - 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, - 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, - 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x8b, 0x01, 0x0a, - 0x0b, 0x4d, 0x61, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x07, - 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x73, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, - 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, - 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, - 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, - 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, - 0x79, 0x32, 0x71, 0x0a, 0x03, 0x4d, 0x61, 0x70, 0x12, 0x30, 0x0a, 0x05, 0x4d, 0x61, 0x70, 0x46, - 0x6e, 0x12, 0x12, 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x70, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4d, - 0x61, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x07, 0x49, 0x73, - 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x15, 0x2e, - 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x37, 0x5a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, - 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x61, 0x70, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x84, 0x04, 0x0a, 0x0a, 0x4d, 0x61, 0x70, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x34, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, + 0x2e, 0x4d, 0x61, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x34, 0x0a, 0x09, + 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x11, 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, + 0x6b, 0x65, 0x48, 0x00, 0x52, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x88, + 0x01, 0x01, 0x12, 0x37, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x61, 0x6e, + 0x73, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x01, + 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x88, 0x01, 0x01, 0x1a, 0xa7, 0x02, 0x0a, 0x07, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, + 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, + 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, + 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x12, 0x41, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, + 0x2e, 0x4d, 0x61, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, + 0x61, 0x6b, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x1d, + 0x0a, 0x09, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x73, + 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x73, 0x6f, 0x74, 0x22, 0x26, 0x0a, + 0x12, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x03, 0x65, 0x6f, 0x74, 0x22, 0xa3, 0x02, 0x0a, 0x0b, 0x4d, 0x61, 0x70, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, + 0x4d, 0x61, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x34, 0x0a, 0x09, 0x68, + 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, + 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, + 0x65, 0x48, 0x00, 0x52, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x88, 0x01, + 0x01, 0x12, 0x37, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x48, 0x01, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x88, 0x01, 0x01, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, + 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, + 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, + 0x67, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, + 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, + 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, + 0x64, 0x79, 0x32, 0x75, 0x0a, 0x03, 0x4d, 0x61, 0x70, 0x12, 0x34, 0x0a, 0x05, 0x4d, 0x61, 0x70, + 0x46, 0x6e, 0x12, 0x12, 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x70, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, + 0x4d, 0x61, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, + 0x38, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, + 0x74, 0x79, 0x1a, 0x15, 0x2e, 0x6d, 0x61, 0x70, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, + 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x37, 0x5a, 0x35, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, + 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, + 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x61, 0x70, 0x2f, + 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -329,30 +551,38 @@ func file_pkg_apis_proto_map_v1_map_proto_rawDescGZIP() []byte { return file_pkg_apis_proto_map_v1_map_proto_rawDescData } -var file_pkg_apis_proto_map_v1_map_proto_msgTypes = make([]protoimpl.MessageInfo, 5) -var file_pkg_apis_proto_map_v1_map_proto_goTypes = []interface{}{ +var file_pkg_apis_proto_map_v1_map_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_pkg_apis_proto_map_v1_map_proto_goTypes = []any{ (*MapRequest)(nil), // 0: map.v1.MapRequest - (*MapResponse)(nil), // 1: map.v1.MapResponse - (*ReadyResponse)(nil), // 2: map.v1.ReadyResponse - nil, // 3: map.v1.MapRequest.HeadersEntry - (*MapResponse_Result)(nil), // 4: map.v1.MapResponse.Result - (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 6: google.protobuf.Empty + (*Handshake)(nil), // 1: map.v1.Handshake + (*TransmissionStatus)(nil), // 2: map.v1.TransmissionStatus + (*MapResponse)(nil), // 3: map.v1.MapResponse + (*ReadyResponse)(nil), // 4: map.v1.ReadyResponse + (*MapRequest_Request)(nil), // 5: map.v1.MapRequest.Request + nil, // 6: map.v1.MapRequest.Request.HeadersEntry + (*MapResponse_Result)(nil), // 7: map.v1.MapResponse.Result + (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 9: google.protobuf.Empty } var file_pkg_apis_proto_map_v1_map_proto_depIdxs = []int32{ - 5, // 0: map.v1.MapRequest.event_time:type_name -> google.protobuf.Timestamp - 5, // 1: map.v1.MapRequest.watermark:type_name -> google.protobuf.Timestamp - 3, // 2: map.v1.MapRequest.headers:type_name -> map.v1.MapRequest.HeadersEntry - 4, // 3: map.v1.MapResponse.results:type_name -> map.v1.MapResponse.Result - 0, // 4: map.v1.Map.MapFn:input_type -> map.v1.MapRequest - 6, // 5: map.v1.Map.IsReady:input_type -> google.protobuf.Empty - 1, // 6: map.v1.Map.MapFn:output_type -> map.v1.MapResponse - 2, // 7: map.v1.Map.IsReady:output_type -> map.v1.ReadyResponse - 6, // [6:8] is the sub-list for method output_type - 4, // [4:6] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 5, // 0: map.v1.MapRequest.request:type_name -> map.v1.MapRequest.Request + 1, // 1: map.v1.MapRequest.handshake:type_name -> map.v1.Handshake + 2, // 2: map.v1.MapRequest.status:type_name -> map.v1.TransmissionStatus + 7, // 3: map.v1.MapResponse.results:type_name -> map.v1.MapResponse.Result + 1, // 4: map.v1.MapResponse.handshake:type_name -> map.v1.Handshake + 2, // 5: map.v1.MapResponse.status:type_name -> map.v1.TransmissionStatus + 8, // 6: map.v1.MapRequest.Request.event_time:type_name -> google.protobuf.Timestamp + 8, // 7: map.v1.MapRequest.Request.watermark:type_name -> google.protobuf.Timestamp + 6, // 8: map.v1.MapRequest.Request.headers:type_name -> map.v1.MapRequest.Request.HeadersEntry + 0, // 9: map.v1.Map.MapFn:input_type -> map.v1.MapRequest + 9, // 10: map.v1.Map.IsReady:input_type -> google.protobuf.Empty + 3, // 11: map.v1.Map.MapFn:output_type -> map.v1.MapResponse + 4, // 12: map.v1.Map.IsReady:output_type -> map.v1.ReadyResponse + 11, // [11:13] is the sub-list for method output_type + 9, // [9:11] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_pkg_apis_proto_map_v1_map_proto_init() } @@ -361,7 +591,7 @@ func file_pkg_apis_proto_map_v1_map_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_pkg_apis_proto_map_v1_map_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_map_v1_map_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*MapRequest); i { case 0: return &v.state @@ -373,7 +603,31 @@ func file_pkg_apis_proto_map_v1_map_proto_init() { return nil } } - file_pkg_apis_proto_map_v1_map_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_map_v1_map_proto_msgTypes[1].Exporter = func(v any, i int) any { + switch v := v.(*Handshake); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_map_v1_map_proto_msgTypes[2].Exporter = func(v any, i int) any { + switch v := v.(*TransmissionStatus); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_map_v1_map_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*MapResponse); i { case 0: return &v.state @@ -385,7 +639,7 @@ func file_pkg_apis_proto_map_v1_map_proto_init() { return nil } } - file_pkg_apis_proto_map_v1_map_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_map_v1_map_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*ReadyResponse); i { case 0: return &v.state @@ -397,7 +651,19 @@ func file_pkg_apis_proto_map_v1_map_proto_init() { return nil } } - file_pkg_apis_proto_map_v1_map_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_map_v1_map_proto_msgTypes[5].Exporter = func(v any, i int) any { + switch v := v.(*MapRequest_Request); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_map_v1_map_proto_msgTypes[7].Exporter = func(v any, i int) any { switch v := v.(*MapResponse_Result); i { case 0: return &v.state @@ -410,13 +676,15 @@ func file_pkg_apis_proto_map_v1_map_proto_init() { } } } + file_pkg_apis_proto_map_v1_map_proto_msgTypes[0].OneofWrappers = []any{} + file_pkg_apis_proto_map_v1_map_proto_msgTypes[3].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_apis_proto_map_v1_map_proto_rawDesc, NumEnums: 0, - NumMessages: 5, + NumMessages: 8, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/apis/proto/map/v1/map.proto b/pkg/apis/proto/map/v1/map.proto index e93a9b01..22d68b5d 100644 --- a/pkg/apis/proto/map/v1/map.proto +++ b/pkg/apis/proto/map/v1/map.proto @@ -9,7 +9,7 @@ package map.v1; service Map { // MapFn applies a function to each map request element. - rpc MapFn(MapRequest) returns (MapResponse); + rpc MapFn(stream MapRequest) returns (stream MapResponse); // IsReady is the heartbeat endpoint for gRPC. rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); @@ -19,11 +19,33 @@ service Map { * MapRequest represents a request element. */ message MapRequest { - repeated string keys = 1; - bytes value = 2; - google.protobuf.Timestamp event_time = 3; - google.protobuf.Timestamp watermark = 4; - map headers = 5; + message Request { + repeated string keys = 1; + bytes value = 2; + google.protobuf.Timestamp event_time = 3; + google.protobuf.Timestamp watermark = 4; + map headers = 5; + } + Request request = 1; + // This ID is used to uniquely identify a map request + string id = 2; + optional Handshake handshake = 3; + optional TransmissionStatus status = 4; +} + +/* + * Handshake message between client and server to indicate the start of transmission. + */ +message Handshake { + // Required field indicating the start of transmission. + bool sot = 1; +} + +/* + * Status message to indicate the status of the message. + */ +message TransmissionStatus { + bool eot = 1; } /** @@ -36,6 +58,10 @@ message MapResponse { repeated string tags = 3; } repeated Result results = 1; + // This ID is used to refer the responses to the request it corresponds to. + string id = 2; + optional Handshake handshake = 3; + optional TransmissionStatus status = 4; } /** diff --git a/pkg/apis/proto/map/v1/map_grpc.pb.go b/pkg/apis/proto/map/v1/map_grpc.pb.go index d3844348..0ee687b7 100644 --- a/pkg/apis/proto/map/v1/map_grpc.pb.go +++ b/pkg/apis/proto/map/v1/map_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v4.25.1 // source: pkg/apis/proto/map/v1/map.proto @@ -16,15 +16,20 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + Map_MapFn_FullMethodName = "/map.v1.Map/MapFn" + Map_IsReady_FullMethodName = "/map.v1.Map/IsReady" +) // MapClient is the client API for Map service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type MapClient interface { // MapFn applies a function to each map request element. - MapFn(ctx context.Context, in *MapRequest, opts ...grpc.CallOption) (*MapResponse, error) + MapFn(ctx context.Context, opts ...grpc.CallOption) (Map_MapFnClient, error) // IsReady is the heartbeat endpoint for gRPC. IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) } @@ -37,18 +42,42 @@ func NewMapClient(cc grpc.ClientConnInterface) MapClient { return &mapClient{cc} } -func (c *mapClient) MapFn(ctx context.Context, in *MapRequest, opts ...grpc.CallOption) (*MapResponse, error) { - out := new(MapResponse) - err := c.cc.Invoke(ctx, "/map.v1.Map/MapFn", in, out, opts...) +func (c *mapClient) MapFn(ctx context.Context, opts ...grpc.CallOption) (Map_MapFnClient, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Map_ServiceDesc.Streams[0], Map_MapFn_FullMethodName, cOpts...) if err != nil { return nil, err } - return out, nil + x := &mapMapFnClient{ClientStream: stream} + return x, nil +} + +type Map_MapFnClient interface { + Send(*MapRequest) error + Recv() (*MapResponse, error) + grpc.ClientStream +} + +type mapMapFnClient struct { + grpc.ClientStream +} + +func (x *mapMapFnClient) Send(m *MapRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *mapMapFnClient) Recv() (*MapResponse, error) { + m := new(MapResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil } func (c *mapClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ReadyResponse) - err := c.cc.Invoke(ctx, "/map.v1.Map/IsReady", in, out, opts...) + err := c.cc.Invoke(ctx, Map_IsReady_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -60,7 +89,7 @@ func (c *mapClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc // for forward compatibility type MapServer interface { // MapFn applies a function to each map request element. - MapFn(context.Context, *MapRequest) (*MapResponse, error) + MapFn(Map_MapFnServer) error // IsReady is the heartbeat endpoint for gRPC. IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) mustEmbedUnimplementedMapServer() @@ -70,8 +99,8 @@ type MapServer interface { type UnimplementedMapServer struct { } -func (UnimplementedMapServer) MapFn(context.Context, *MapRequest) (*MapResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method MapFn not implemented") +func (UnimplementedMapServer) MapFn(Map_MapFnServer) error { + return status.Errorf(codes.Unimplemented, "method MapFn not implemented") } func (UnimplementedMapServer) IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method IsReady not implemented") @@ -89,22 +118,30 @@ func RegisterMapServer(s grpc.ServiceRegistrar, srv MapServer) { s.RegisterService(&Map_ServiceDesc, srv) } -func _Map_MapFn_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(MapRequest) - if err := dec(in); err != nil { +func _Map_MapFn_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(MapServer).MapFn(&mapMapFnServer{ServerStream: stream}) +} + +type Map_MapFnServer interface { + Send(*MapResponse) error + Recv() (*MapRequest, error) + grpc.ServerStream +} + +type mapMapFnServer struct { + grpc.ServerStream +} + +func (x *mapMapFnServer) Send(m *MapResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *mapMapFnServer) Recv() (*MapRequest, error) { + m := new(MapRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { return nil, err } - if interceptor == nil { - return srv.(MapServer).MapFn(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/map.v1.Map/MapFn", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MapServer).MapFn(ctx, req.(*MapRequest)) - } - return interceptor(ctx, in, info, handler) + return m, nil } func _Map_IsReady_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { @@ -117,7 +154,7 @@ func _Map_IsReady_Handler(srv interface{}, ctx context.Context, dec func(interfa } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/map.v1.Map/IsReady", + FullMethod: Map_IsReady_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(MapServer).IsReady(ctx, req.(*emptypb.Empty)) @@ -132,15 +169,18 @@ var Map_ServiceDesc = grpc.ServiceDesc{ ServiceName: "map.v1.Map", HandlerType: (*MapServer)(nil), Methods: []grpc.MethodDesc{ - { - MethodName: "MapFn", - Handler: _Map_MapFn_Handler, - }, { MethodName: "IsReady", Handler: _Map_IsReady_Handler, }, }, - Streams: []grpc.StreamDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "MapFn", + Handler: _Map_MapFn_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, Metadata: "pkg/apis/proto/map/v1/map.proto", } diff --git a/pkg/apis/proto/mapstream/v1/mapstream.pb.go b/pkg/apis/proto/mapstream/v1/mapstream.pb.go deleted file mode 100644 index 6a276ae2..00000000 --- a/pkg/apis/proto/mapstream/v1/mapstream.pb.go +++ /dev/null @@ -1,437 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// versions: -// protoc-gen-go v1.33.0 -// protoc v4.25.1 -// source: pkg/apis/proto/mapstream/v1/mapstream.proto - -package v1 - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - emptypb "google.golang.org/protobuf/types/known/emptypb" - timestamppb "google.golang.org/protobuf/types/known/timestamppb" - reflect "reflect" - sync "sync" -) - -const ( - // Verify that this generated code is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) - // Verify that runtime/protoimpl is sufficiently up-to-date. - _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) -) - -// * -// MapStreamRequest represents a request element. -type MapStreamRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` - Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` - Headers map[string]string `protobuf:"bytes,5,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` -} - -func (x *MapStreamRequest) Reset() { - *x = MapStreamRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MapStreamRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MapStreamRequest) ProtoMessage() {} - -func (x *MapStreamRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MapStreamRequest.ProtoReflect.Descriptor instead. -func (*MapStreamRequest) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDescGZIP(), []int{0} -} - -func (x *MapStreamRequest) GetKeys() []string { - if x != nil { - return x.Keys - } - return nil -} - -func (x *MapStreamRequest) GetValue() []byte { - if x != nil { - return x.Value - } - return nil -} - -func (x *MapStreamRequest) GetEventTime() *timestamppb.Timestamp { - if x != nil { - return x.EventTime - } - return nil -} - -func (x *MapStreamRequest) GetWatermark() *timestamppb.Timestamp { - if x != nil { - return x.Watermark - } - return nil -} - -func (x *MapStreamRequest) GetHeaders() map[string]string { - if x != nil { - return x.Headers - } - return nil -} - -// * -// MapStreamResponse represents a response element. -type MapStreamResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Result *MapStreamResponse_Result `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` -} - -func (x *MapStreamResponse) Reset() { - *x = MapStreamResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MapStreamResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MapStreamResponse) ProtoMessage() {} - -func (x *MapStreamResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MapStreamResponse.ProtoReflect.Descriptor instead. -func (*MapStreamResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDescGZIP(), []int{1} -} - -func (x *MapStreamResponse) GetResult() *MapStreamResponse_Result { - if x != nil { - return x.Result - } - return nil -} - -// * -// ReadyResponse is the health check result. -type ReadyResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` -} - -func (x *ReadyResponse) Reset() { - *x = ReadyResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ReadyResponse) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ReadyResponse) ProtoMessage() {} - -func (x *ReadyResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ReadyResponse.ProtoReflect.Descriptor instead. -func (*ReadyResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDescGZIP(), []int{2} -} - -func (x *ReadyResponse) GetReady() bool { - if x != nil { - return x.Ready - } - return false -} - -type MapStreamResponse_Result struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - Tags []string `protobuf:"bytes,3,rep,name=tags,proto3" json:"tags,omitempty"` -} - -func (x *MapStreamResponse_Result) Reset() { - *x = MapStreamResponse_Result{} - if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *MapStreamResponse_Result) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*MapStreamResponse_Result) ProtoMessage() {} - -func (x *MapStreamResponse_Result) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use MapStreamResponse_Result.ProtoReflect.Descriptor instead. -func (*MapStreamResponse_Result) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDescGZIP(), []int{1, 0} -} - -func (x *MapStreamResponse_Result) GetKeys() []string { - if x != nil { - return x.Keys - } - return nil -} - -func (x *MapStreamResponse_Result) GetValue() []byte { - if x != nil { - return x.Value - } - return nil -} - -func (x *MapStreamResponse_Result) GetTags() []string { - if x != nil { - return x.Tags - } - return nil -} - -var File_pkg_apis_proto_mapstream_v1_mapstream_proto protoreflect.FileDescriptor - -var file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDesc = []byte{ - 0x0a, 0x2b, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x2f, 0x6d, 0x61, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2f, 0x76, 0x31, 0x2f, 0x6d, 0x61, - 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0c, 0x6d, - 0x61, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76, 0x31, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, - 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb4, 0x02, 0x0a, 0x10, 0x4d, 0x61, - 0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, - 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, - 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, - 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, - 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x12, 0x45, 0x0a, - 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, - 0x2e, 0x6d, 0x61, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, - 0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x48, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x73, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, - 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0x9b, 0x01, 0x0a, 0x11, 0x4d, 0x61, 0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6d, 0x61, 0x70, 0x73, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, - 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x1a, 0x46, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, - 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, - 0x67, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x22, 0x25, - 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, - 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0x9d, 0x01, 0x0a, 0x09, 0x4d, 0x61, 0x70, 0x53, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x12, 0x50, 0x0a, 0x0b, 0x4d, 0x61, 0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, - 0x46, 0x6e, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76, - 0x31, 0x2e, 0x4d, 0x61, 0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76, - 0x31, 0x2e, 0x4d, 0x61, 0x70, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x3e, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, - 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1b, 0x2e, 0x6d, 0x61, 0x70, 0x73, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, - 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, - 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x6d, 0x61, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, -} - -var ( - file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDescOnce sync.Once - file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDescData = file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDesc -) - -func file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDescGZIP() []byte { - file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDescOnce.Do(func() { - file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDescData) - }) - return file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDescData -} - -var file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes = make([]protoimpl.MessageInfo, 5) -var file_pkg_apis_proto_mapstream_v1_mapstream_proto_goTypes = []interface{}{ - (*MapStreamRequest)(nil), // 0: mapstream.v1.MapStreamRequest - (*MapStreamResponse)(nil), // 1: mapstream.v1.MapStreamResponse - (*ReadyResponse)(nil), // 2: mapstream.v1.ReadyResponse - nil, // 3: mapstream.v1.MapStreamRequest.HeadersEntry - (*MapStreamResponse_Result)(nil), // 4: mapstream.v1.MapStreamResponse.Result - (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 6: google.protobuf.Empty -} -var file_pkg_apis_proto_mapstream_v1_mapstream_proto_depIdxs = []int32{ - 5, // 0: mapstream.v1.MapStreamRequest.event_time:type_name -> google.protobuf.Timestamp - 5, // 1: mapstream.v1.MapStreamRequest.watermark:type_name -> google.protobuf.Timestamp - 3, // 2: mapstream.v1.MapStreamRequest.headers:type_name -> mapstream.v1.MapStreamRequest.HeadersEntry - 4, // 3: mapstream.v1.MapStreamResponse.result:type_name -> mapstream.v1.MapStreamResponse.Result - 0, // 4: mapstream.v1.MapStream.MapStreamFn:input_type -> mapstream.v1.MapStreamRequest - 6, // 5: mapstream.v1.MapStream.IsReady:input_type -> google.protobuf.Empty - 1, // 6: mapstream.v1.MapStream.MapStreamFn:output_type -> mapstream.v1.MapStreamResponse - 2, // 7: mapstream.v1.MapStream.IsReady:output_type -> mapstream.v1.ReadyResponse - 6, // [6:8] is the sub-list for method output_type - 4, // [4:6] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name -} - -func init() { file_pkg_apis_proto_mapstream_v1_mapstream_proto_init() } -func file_pkg_apis_proto_mapstream_v1_mapstream_proto_init() { - if File_pkg_apis_proto_mapstream_v1_mapstream_proto != nil { - return - } - if !protoimpl.UnsafeEnabled { - file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MapStreamRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MapStreamResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ReadyResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*MapStreamResponse_Result); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDesc, - NumEnums: 0, - NumMessages: 5, - NumExtensions: 0, - NumServices: 1, - }, - GoTypes: file_pkg_apis_proto_mapstream_v1_mapstream_proto_goTypes, - DependencyIndexes: file_pkg_apis_proto_mapstream_v1_mapstream_proto_depIdxs, - MessageInfos: file_pkg_apis_proto_mapstream_v1_mapstream_proto_msgTypes, - }.Build() - File_pkg_apis_proto_mapstream_v1_mapstream_proto = out.File - file_pkg_apis_proto_mapstream_v1_mapstream_proto_rawDesc = nil - file_pkg_apis_proto_mapstream_v1_mapstream_proto_goTypes = nil - file_pkg_apis_proto_mapstream_v1_mapstream_proto_depIdxs = nil -} diff --git a/pkg/apis/proto/mapstream/v1/mapstream.proto b/pkg/apis/proto/mapstream/v1/mapstream.proto deleted file mode 100644 index 43f85f60..00000000 --- a/pkg/apis/proto/mapstream/v1/mapstream.proto +++ /dev/null @@ -1,47 +0,0 @@ -syntax = "proto3"; - -option go_package = "github.com/numaproj/numaflow-go/pkg/apis/proto/mapstream/v1"; - -import "google/protobuf/empty.proto"; -import "google/protobuf/timestamp.proto"; - - -package mapstream.v1; - -service MapStream { - // MapStreamFn applies a function to each request element and returns a stream. - rpc MapStreamFn(MapStreamRequest) returns (stream MapStreamResponse); - - // IsReady is the heartbeat endpoint for gRPC. - rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); -} - -/** - * MapStreamRequest represents a request element. - */ -message MapStreamRequest { - repeated string keys = 1; - bytes value = 2; - google.protobuf.Timestamp event_time = 3; - google.protobuf.Timestamp watermark = 4; - map headers = 5; -} - -/** - * MapStreamResponse represents a response element. - */ -message MapStreamResponse { - message Result { - repeated string keys = 1; - bytes value = 2; - repeated string tags = 3; - } - Result result = 1; -} - -/** - * ReadyResponse is the health check result. - */ -message ReadyResponse { - bool ready = 1; -} \ No newline at end of file diff --git a/pkg/apis/proto/mapstream/v1/mapstream_grpc.pb.go b/pkg/apis/proto/mapstream/v1/mapstream_grpc.pb.go deleted file mode 100644 index e8927dc5..00000000 --- a/pkg/apis/proto/mapstream/v1/mapstream_grpc.pb.go +++ /dev/null @@ -1,174 +0,0 @@ -// Code generated by protoc-gen-go-grpc. DO NOT EDIT. -// versions: -// - protoc-gen-go-grpc v1.2.0 -// - protoc v4.25.1 -// source: pkg/apis/proto/mapstream/v1/mapstream.proto - -package v1 - -import ( - context "context" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" - emptypb "google.golang.org/protobuf/types/known/emptypb" -) - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 - -// MapStreamClient is the client API for MapStream service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. -type MapStreamClient interface { - // MapStreamFn applies a function to each request element and returns a stream. - MapStreamFn(ctx context.Context, in *MapStreamRequest, opts ...grpc.CallOption) (MapStream_MapStreamFnClient, error) - // IsReady is the heartbeat endpoint for gRPC. - IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) -} - -type mapStreamClient struct { - cc grpc.ClientConnInterface -} - -func NewMapStreamClient(cc grpc.ClientConnInterface) MapStreamClient { - return &mapStreamClient{cc} -} - -func (c *mapStreamClient) MapStreamFn(ctx context.Context, in *MapStreamRequest, opts ...grpc.CallOption) (MapStream_MapStreamFnClient, error) { - stream, err := c.cc.NewStream(ctx, &MapStream_ServiceDesc.Streams[0], "/mapstream.v1.MapStream/MapStreamFn", opts...) - if err != nil { - return nil, err - } - x := &mapStreamMapStreamFnClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type MapStream_MapStreamFnClient interface { - Recv() (*MapStreamResponse, error) - grpc.ClientStream -} - -type mapStreamMapStreamFnClient struct { - grpc.ClientStream -} - -func (x *mapStreamMapStreamFnClient) Recv() (*MapStreamResponse, error) { - m := new(MapStreamResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -func (c *mapStreamClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { - out := new(ReadyResponse) - err := c.cc.Invoke(ctx, "/mapstream.v1.MapStream/IsReady", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -// MapStreamServer is the server API for MapStream service. -// All implementations must embed UnimplementedMapStreamServer -// for forward compatibility -type MapStreamServer interface { - // MapStreamFn applies a function to each request element and returns a stream. - MapStreamFn(*MapStreamRequest, MapStream_MapStreamFnServer) error - // IsReady is the heartbeat endpoint for gRPC. - IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) - mustEmbedUnimplementedMapStreamServer() -} - -// UnimplementedMapStreamServer must be embedded to have forward compatible implementations. -type UnimplementedMapStreamServer struct { -} - -func (UnimplementedMapStreamServer) MapStreamFn(*MapStreamRequest, MapStream_MapStreamFnServer) error { - return status.Errorf(codes.Unimplemented, "method MapStreamFn not implemented") -} -func (UnimplementedMapStreamServer) IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method IsReady not implemented") -} -func (UnimplementedMapStreamServer) mustEmbedUnimplementedMapStreamServer() {} - -// UnsafeMapStreamServer may be embedded to opt out of forward compatibility for this service. -// Use of this interface is not recommended, as added methods to MapStreamServer will -// result in compilation errors. -type UnsafeMapStreamServer interface { - mustEmbedUnimplementedMapStreamServer() -} - -func RegisterMapStreamServer(s grpc.ServiceRegistrar, srv MapStreamServer) { - s.RegisterService(&MapStream_ServiceDesc, srv) -} - -func _MapStream_MapStreamFn_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(MapStreamRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(MapStreamServer).MapStreamFn(m, &mapStreamMapStreamFnServer{stream}) -} - -type MapStream_MapStreamFnServer interface { - Send(*MapStreamResponse) error - grpc.ServerStream -} - -type mapStreamMapStreamFnServer struct { - grpc.ServerStream -} - -func (x *mapStreamMapStreamFnServer) Send(m *MapStreamResponse) error { - return x.ServerStream.SendMsg(m) -} - -func _MapStream_IsReady_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(emptypb.Empty) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(MapStreamServer).IsReady(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/mapstream.v1.MapStream/IsReady", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(MapStreamServer).IsReady(ctx, req.(*emptypb.Empty)) - } - return interceptor(ctx, in, info, handler) -} - -// MapStream_ServiceDesc is the grpc.ServiceDesc for MapStream service. -// It's only intended for direct use with grpc.RegisterService, -// and not to be introspected or modified (even as a copy) -var MapStream_ServiceDesc = grpc.ServiceDesc{ - ServiceName: "mapstream.v1.MapStream", - HandlerType: (*MapStreamServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "IsReady", - Handler: _MapStream_IsReady_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "MapStreamFn", - Handler: _MapStream_MapStreamFn_Handler, - ServerStreams: true, - }, - }, - Metadata: "pkg/apis/proto/mapstream/v1/mapstream.proto", -} diff --git a/pkg/apis/proto/mapstream/v1/mapstreammock/mapstreammock.go b/pkg/apis/proto/mapstream/v1/mapstreammock/mapstreammock.go deleted file mode 100644 index 025080bb..00000000 --- a/pkg/apis/proto/mapstream/v1/mapstreammock/mapstreammock.go +++ /dev/null @@ -1,202 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/numaproj/numaflow-go/pkg/apis/proto/mapstream/v1 (interfaces: MapStreamClient,MapStream_MapStreamFnClient) - -// Package mapstreammock is a generated GoMock package. -package mapstreammock - -import ( - context "context" - reflect "reflect" - - gomock "github.com/golang/mock/gomock" - v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/mapstream/v1" - grpc "google.golang.org/grpc" - metadata "google.golang.org/grpc/metadata" - emptypb "google.golang.org/protobuf/types/known/emptypb" -) - -// MockMapStreamClient is a mock of MapStreamClient interface. -type MockMapStreamClient struct { - ctrl *gomock.Controller - recorder *MockMapStreamClientMockRecorder -} - -// MockMapStreamClientMockRecorder is the mock recorder for MockMapStreamClient. -type MockMapStreamClientMockRecorder struct { - mock *MockMapStreamClient -} - -// NewMockMapStreamClient creates a new mock instance. -func NewMockMapStreamClient(ctrl *gomock.Controller) *MockMapStreamClient { - mock := &MockMapStreamClient{ctrl: ctrl} - mock.recorder = &MockMapStreamClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockMapStreamClient) EXPECT() *MockMapStreamClientMockRecorder { - return m.recorder -} - -// IsReady mocks base method. -func (m *MockMapStreamClient) IsReady(arg0 context.Context, arg1 *emptypb.Empty, arg2 ...grpc.CallOption) (*v1.ReadyResponse, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "IsReady", varargs...) - ret0, _ := ret[0].(*v1.ReadyResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// IsReady indicates an expected call of IsReady. -func (mr *MockMapStreamClientMockRecorder) IsReady(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsReady", reflect.TypeOf((*MockMapStreamClient)(nil).IsReady), varargs...) -} - -// MapStreamFn mocks base method. -func (m *MockMapStreamClient) MapStreamFn(arg0 context.Context, arg1 *v1.MapStreamRequest, arg2 ...grpc.CallOption) (v1.MapStream_MapStreamFnClient, error) { - m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "MapStreamFn", varargs...) - ret0, _ := ret[0].(v1.MapStream_MapStreamFnClient) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// MapStreamFn indicates an expected call of MapStreamFn. -func (mr *MockMapStreamClientMockRecorder) MapStreamFn(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MapStreamFn", reflect.TypeOf((*MockMapStreamClient)(nil).MapStreamFn), varargs...) -} - -// MockMapStream_MapStreamFnClient is a mock of MapStream_MapStreamFnClient interface. -type MockMapStream_MapStreamFnClient struct { - ctrl *gomock.Controller - recorder *MockMapStream_MapStreamFnClientMockRecorder -} - -// MockMapStream_MapStreamFnClientMockRecorder is the mock recorder for MockMapStream_MapStreamFnClient. -type MockMapStream_MapStreamFnClientMockRecorder struct { - mock *MockMapStream_MapStreamFnClient -} - -// NewMockMapStream_MapStreamFnClient creates a new mock instance. -func NewMockMapStream_MapStreamFnClient(ctrl *gomock.Controller) *MockMapStream_MapStreamFnClient { - mock := &MockMapStream_MapStreamFnClient{ctrl: ctrl} - mock.recorder = &MockMapStream_MapStreamFnClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockMapStream_MapStreamFnClient) EXPECT() *MockMapStream_MapStreamFnClientMockRecorder { - return m.recorder -} - -// CloseSend mocks base method. -func (m *MockMapStream_MapStreamFnClient) CloseSend() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CloseSend") - ret0, _ := ret[0].(error) - return ret0 -} - -// CloseSend indicates an expected call of CloseSend. -func (mr *MockMapStream_MapStreamFnClientMockRecorder) CloseSend() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseSend", reflect.TypeOf((*MockMapStream_MapStreamFnClient)(nil).CloseSend)) -} - -// Context mocks base method. -func (m *MockMapStream_MapStreamFnClient) Context() context.Context { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Context") - ret0, _ := ret[0].(context.Context) - return ret0 -} - -// Context indicates an expected call of Context. -func (mr *MockMapStream_MapStreamFnClientMockRecorder) Context() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Context", reflect.TypeOf((*MockMapStream_MapStreamFnClient)(nil).Context)) -} - -// Header mocks base method. -func (m *MockMapStream_MapStreamFnClient) Header() (metadata.MD, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Header") - ret0, _ := ret[0].(metadata.MD) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Header indicates an expected call of Header. -func (mr *MockMapStream_MapStreamFnClientMockRecorder) Header() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Header", reflect.TypeOf((*MockMapStream_MapStreamFnClient)(nil).Header)) -} - -// Recv mocks base method. -func (m *MockMapStream_MapStreamFnClient) Recv() (*v1.MapStreamResponse, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Recv") - ret0, _ := ret[0].(*v1.MapStreamResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Recv indicates an expected call of Recv. -func (mr *MockMapStream_MapStreamFnClientMockRecorder) Recv() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Recv", reflect.TypeOf((*MockMapStream_MapStreamFnClient)(nil).Recv)) -} - -// RecvMsg mocks base method. -func (m *MockMapStream_MapStreamFnClient) RecvMsg(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RecvMsg", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// RecvMsg indicates an expected call of RecvMsg. -func (mr *MockMapStream_MapStreamFnClientMockRecorder) RecvMsg(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockMapStream_MapStreamFnClient)(nil).RecvMsg), arg0) -} - -// SendMsg mocks base method. -func (m *MockMapStream_MapStreamFnClient) SendMsg(arg0 interface{}) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendMsg", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendMsg indicates an expected call of SendMsg. -func (mr *MockMapStream_MapStreamFnClientMockRecorder) SendMsg(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockMapStream_MapStreamFnClient)(nil).SendMsg), arg0) -} - -// Trailer mocks base method. -func (m *MockMapStream_MapStreamFnClient) Trailer() metadata.MD { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Trailer") - ret0, _ := ret[0].(metadata.MD) - return ret0 -} - -// Trailer indicates an expected call of Trailer. -func (mr *MockMapStream_MapStreamFnClientMockRecorder) Trailer() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Trailer", reflect.TypeOf((*MockMapStream_MapStreamFnClient)(nil).Trailer)) -} diff --git a/pkg/apis/proto/mapstream/v1/mockgen.go b/pkg/apis/proto/mapstream/v1/mockgen.go deleted file mode 100644 index f75e0855..00000000 --- a/pkg/apis/proto/mapstream/v1/mockgen.go +++ /dev/null @@ -1,3 +0,0 @@ -package v1 - -//go:generate mockgen -destination mapstreammock/mapstreammock.go -package mapstreammock github.com/numaproj/numaflow-go/pkg/apis/proto/mapstream/v1 MapStreamClient,MapStream_MapStreamFnClient diff --git a/pkg/apis/proto/reduce/v1/reduce.pb.go b/pkg/apis/proto/reduce/v1/reduce.pb.go index 597d4c7c..2563e299 100644 --- a/pkg/apis/proto/reduce/v1/reduce.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.34.2 // protoc v4.25.1 // source: pkg/apis/proto/reduce/v1/reduce.proto @@ -612,7 +612,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_rawDescGZIP() []byte { var file_pkg_apis_proto_reduce_v1_reduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes = make([]protoimpl.MessageInfo, 8) -var file_pkg_apis_proto_reduce_v1_reduce_proto_goTypes = []interface{}{ +var file_pkg_apis_proto_reduce_v1_reduce_proto_goTypes = []any{ (ReduceRequest_WindowOperation_Event)(0), // 0: reduce.v1.ReduceRequest.WindowOperation.Event (*ReduceRequest)(nil), // 1: reduce.v1.ReduceRequest (*Window)(nil), // 2: reduce.v1.Window @@ -654,7 +654,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*ReduceRequest); i { case 0: return &v.state @@ -666,7 +666,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { return nil } } - file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*Window); i { case 0: return &v.state @@ -678,7 +678,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { return nil } } - file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*ReduceResponse); i { case 0: return &v.state @@ -690,7 +690,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { return nil } } - file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*ReadyResponse); i { case 0: return &v.state @@ -702,7 +702,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { return nil } } - file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*ReduceRequest_WindowOperation); i { case 0: return &v.state @@ -714,7 +714,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { return nil } } - file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*ReduceRequest_Payload); i { case 0: return &v.state @@ -726,7 +726,7 @@ func file_pkg_apis_proto_reduce_v1_reduce_proto_init() { return nil } } - file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_reduce_v1_reduce_proto_msgTypes[7].Exporter = func(v any, i int) any { switch v := v.(*ReduceResponse_Result); i { case 0: return &v.state diff --git a/pkg/apis/proto/reduce/v1/reduce_grpc.pb.go b/pkg/apis/proto/reduce/v1/reduce_grpc.pb.go index afe493d1..21d2a3d0 100644 --- a/pkg/apis/proto/reduce/v1/reduce_grpc.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v4.25.1 // source: pkg/apis/proto/reduce/v1/reduce.proto @@ -16,8 +16,13 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + Reduce_ReduceFn_FullMethodName = "/reduce.v1.Reduce/ReduceFn" + Reduce_IsReady_FullMethodName = "/reduce.v1.Reduce/IsReady" +) // ReduceClient is the client API for Reduce service. // @@ -38,11 +43,12 @@ func NewReduceClient(cc grpc.ClientConnInterface) ReduceClient { } func (c *reduceClient) ReduceFn(ctx context.Context, opts ...grpc.CallOption) (Reduce_ReduceFnClient, error) { - stream, err := c.cc.NewStream(ctx, &Reduce_ServiceDesc.Streams[0], "/reduce.v1.Reduce/ReduceFn", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Reduce_ServiceDesc.Streams[0], Reduce_ReduceFn_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &reduceReduceFnClient{stream} + x := &reduceReduceFnClient{ClientStream: stream} return x, nil } @@ -69,8 +75,9 @@ func (x *reduceReduceFnClient) Recv() (*ReduceResponse, error) { } func (c *reduceClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ReadyResponse) - err := c.cc.Invoke(ctx, "/reduce.v1.Reduce/IsReady", in, out, opts...) + err := c.cc.Invoke(ctx, Reduce_IsReady_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -112,7 +119,7 @@ func RegisterReduceServer(s grpc.ServiceRegistrar, srv ReduceServer) { } func _Reduce_ReduceFn_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(ReduceServer).ReduceFn(&reduceReduceFnServer{stream}) + return srv.(ReduceServer).ReduceFn(&reduceReduceFnServer{ServerStream: stream}) } type Reduce_ReduceFnServer interface { @@ -147,7 +154,7 @@ func _Reduce_IsReady_Handler(srv interface{}, ctx context.Context, dec func(inte } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/reduce.v1.Reduce/IsReady", + FullMethod: Reduce_IsReady_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(ReduceServer).IsReady(ctx, req.(*emptypb.Empty)) diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go index fe19a9ae..57b171b1 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.34.2 // protoc v4.25.1 // source: pkg/apis/proto/sessionreduce/v1/sessionreduce.proto @@ -642,7 +642,7 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_rawDescGZIP() []by var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes = make([]protoimpl.MessageInfo, 8) -var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_goTypes = []interface{}{ +var file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_goTypes = []any{ (SessionReduceRequest_WindowOperation_Event)(0), // 0: sessionreduce.v1.SessionReduceRequest.WindowOperation.Event (*KeyedWindow)(nil), // 1: sessionreduce.v1.KeyedWindow (*SessionReduceRequest)(nil), // 2: sessionreduce.v1.SessionReduceRequest @@ -684,7 +684,7 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*KeyedWindow); i { case 0: return &v.state @@ -696,7 +696,7 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() { return nil } } - file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*SessionReduceRequest); i { case 0: return &v.state @@ -708,7 +708,7 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() { return nil } } - file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*SessionReduceResponse); i { case 0: return &v.state @@ -720,7 +720,7 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() { return nil } } - file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*ReadyResponse); i { case 0: return &v.state @@ -732,7 +732,7 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() { return nil } } - file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*SessionReduceRequest_WindowOperation); i { case 0: return &v.state @@ -744,7 +744,7 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() { return nil } } - file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*SessionReduceRequest_Payload); i { case 0: return &v.state @@ -756,7 +756,7 @@ func file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_init() { return nil } } - file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sessionreduce_v1_sessionreduce_proto_msgTypes[7].Exporter = func(v any, i int) any { switch v := v.(*SessionReduceResponse_Result); i { case 0: return &v.state diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go index 51b7bd6c..95b3d4e9 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v4.25.1 // source: pkg/apis/proto/sessionreduce/v1/sessionreduce.proto @@ -16,8 +16,13 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + SessionReduce_SessionReduceFn_FullMethodName = "/sessionreduce.v1.SessionReduce/SessionReduceFn" + SessionReduce_IsReady_FullMethodName = "/sessionreduce.v1.SessionReduce/IsReady" +) // SessionReduceClient is the client API for SessionReduce service. // @@ -38,11 +43,12 @@ func NewSessionReduceClient(cc grpc.ClientConnInterface) SessionReduceClient { } func (c *sessionReduceClient) SessionReduceFn(ctx context.Context, opts ...grpc.CallOption) (SessionReduce_SessionReduceFnClient, error) { - stream, err := c.cc.NewStream(ctx, &SessionReduce_ServiceDesc.Streams[0], "/sessionreduce.v1.SessionReduce/SessionReduceFn", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &SessionReduce_ServiceDesc.Streams[0], SessionReduce_SessionReduceFn_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &sessionReduceSessionReduceFnClient{stream} + x := &sessionReduceSessionReduceFnClient{ClientStream: stream} return x, nil } @@ -69,8 +75,9 @@ func (x *sessionReduceSessionReduceFnClient) Recv() (*SessionReduceResponse, err } func (c *sessionReduceClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ReadyResponse) - err := c.cc.Invoke(ctx, "/sessionreduce.v1.SessionReduce/IsReady", in, out, opts...) + err := c.cc.Invoke(ctx, SessionReduce_IsReady_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -112,7 +119,7 @@ func RegisterSessionReduceServer(s grpc.ServiceRegistrar, srv SessionReduceServe } func _SessionReduce_SessionReduceFn_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(SessionReduceServer).SessionReduceFn(&sessionReduceSessionReduceFnServer{stream}) + return srv.(SessionReduceServer).SessionReduceFn(&sessionReduceSessionReduceFnServer{ServerStream: stream}) } type SessionReduce_SessionReduceFnServer interface { @@ -147,7 +154,7 @@ func _SessionReduce_IsReady_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/sessionreduce.v1.SessionReduce/IsReady", + FullMethod: SessionReduce_IsReady_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(SessionReduceServer).IsReady(ctx, req.(*emptypb.Empty)) diff --git a/pkg/apis/proto/sideinput/v1/sideinput.pb.go b/pkg/apis/proto/sideinput/v1/sideinput.pb.go index a117b3a1..b58bab15 100644 --- a/pkg/apis/proto/sideinput/v1/sideinput.pb.go +++ b/pkg/apis/proto/sideinput/v1/sideinput.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.34.2 // protoc v4.25.1 // source: pkg/apis/proto/sideinput/v1/sideinput.proto @@ -176,7 +176,7 @@ func file_pkg_apis_proto_sideinput_v1_sideinput_proto_rawDescGZIP() []byte { } var file_pkg_apis_proto_sideinput_v1_sideinput_proto_msgTypes = make([]protoimpl.MessageInfo, 2) -var file_pkg_apis_proto_sideinput_v1_sideinput_proto_goTypes = []interface{}{ +var file_pkg_apis_proto_sideinput_v1_sideinput_proto_goTypes = []any{ (*SideInputResponse)(nil), // 0: sideinput.v1.SideInputResponse (*ReadyResponse)(nil), // 1: sideinput.v1.ReadyResponse (*emptypb.Empty)(nil), // 2: google.protobuf.Empty @@ -199,7 +199,7 @@ func file_pkg_apis_proto_sideinput_v1_sideinput_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_pkg_apis_proto_sideinput_v1_sideinput_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sideinput_v1_sideinput_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*SideInputResponse); i { case 0: return &v.state @@ -211,7 +211,7 @@ func file_pkg_apis_proto_sideinput_v1_sideinput_proto_init() { return nil } } - file_pkg_apis_proto_sideinput_v1_sideinput_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sideinput_v1_sideinput_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*ReadyResponse); i { case 0: return &v.state diff --git a/pkg/apis/proto/sideinput/v1/sideinput_grpc.pb.go b/pkg/apis/proto/sideinput/v1/sideinput_grpc.pb.go index 2562314d..c80e57ed 100644 --- a/pkg/apis/proto/sideinput/v1/sideinput_grpc.pb.go +++ b/pkg/apis/proto/sideinput/v1/sideinput_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v4.25.1 // source: pkg/apis/proto/sideinput/v1/sideinput.proto @@ -16,12 +16,26 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + SideInput_RetrieveSideInput_FullMethodName = "/sideinput.v1.SideInput/RetrieveSideInput" + SideInput_IsReady_FullMethodName = "/sideinput.v1.SideInput/IsReady" +) // SideInputClient is the client API for SideInput service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// SideInput is the gRPC service for user-defined Side Inputs. +// It is used to propagate changes in the values of the provided Side Inputs +// which allows access to slow updated data or configuration without needing to retrieve +// it during each message processing. +// Through this service we should should be able to:- +// 1. Invoke retrieval request for a single Side Input parameter, which in turn should +// check for updates and return its latest value. +// 2. Provide a health check endpoint to indicate whether the service is ready to be used. type SideInputClient interface { // RetrieveSideInput is the endpoint to retrieve the latest value of a given Side Input. RetrieveSideInput(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*SideInputResponse, error) @@ -38,8 +52,9 @@ func NewSideInputClient(cc grpc.ClientConnInterface) SideInputClient { } func (c *sideInputClient) RetrieveSideInput(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*SideInputResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(SideInputResponse) - err := c.cc.Invoke(ctx, "/sideinput.v1.SideInput/RetrieveSideInput", in, out, opts...) + err := c.cc.Invoke(ctx, SideInput_RetrieveSideInput_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -47,8 +62,9 @@ func (c *sideInputClient) RetrieveSideInput(ctx context.Context, in *emptypb.Emp } func (c *sideInputClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ReadyResponse) - err := c.cc.Invoke(ctx, "/sideinput.v1.SideInput/IsReady", in, out, opts...) + err := c.cc.Invoke(ctx, SideInput_IsReady_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -58,6 +74,15 @@ func (c *sideInputClient) IsReady(ctx context.Context, in *emptypb.Empty, opts . // SideInputServer is the server API for SideInput service. // All implementations must embed UnimplementedSideInputServer // for forward compatibility +// +// SideInput is the gRPC service for user-defined Side Inputs. +// It is used to propagate changes in the values of the provided Side Inputs +// which allows access to slow updated data or configuration without needing to retrieve +// it during each message processing. +// Through this service we should should be able to:- +// 1. Invoke retrieval request for a single Side Input parameter, which in turn should +// check for updates and return its latest value. +// 2. Provide a health check endpoint to indicate whether the service is ready to be used. type SideInputServer interface { // RetrieveSideInput is the endpoint to retrieve the latest value of a given Side Input. RetrieveSideInput(context.Context, *emptypb.Empty) (*SideInputResponse, error) @@ -99,7 +124,7 @@ func _SideInput_RetrieveSideInput_Handler(srv interface{}, ctx context.Context, } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/sideinput.v1.SideInput/RetrieveSideInput", + FullMethod: SideInput_RetrieveSideInput_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(SideInputServer).RetrieveSideInput(ctx, req.(*emptypb.Empty)) @@ -117,7 +142,7 @@ func _SideInput_IsReady_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/sideinput.v1.SideInput/IsReady", + FullMethod: SideInput_IsReady_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(SideInputServer).IsReady(ctx, req.(*emptypb.Empty)) diff --git a/pkg/apis/proto/sink/v1/sink.pb.go b/pkg/apis/proto/sink/v1/sink.pb.go index 3f0dae1f..8a6b051e 100644 --- a/pkg/apis/proto/sink/v1/sink.pb.go +++ b/pkg/apis/proto/sink/v1/sink.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.34.2 // protoc v4.25.1 // source: pkg/apis/proto/sink/v1/sink.proto @@ -79,12 +79,13 @@ type SinkRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` - Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` - Id string `protobuf:"bytes,5,opt,name=id,proto3" json:"id,omitempty"` - Headers map[string]string `protobuf:"bytes,6,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Required field indicating the request. + Request *SinkRequest_Request `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` + // Required field indicating the status of the request. + // If eot is set to true, it indicates the end of transmission. + Status *TransmissionStatus `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` + // optional field indicating the handshake message. + Handshake *Handshake `protobuf:"bytes,3,opt,name=handshake,proto3,oneof" json:"handshake,omitempty"` } func (x *SinkRequest) Reset() { @@ -119,46 +120,74 @@ func (*SinkRequest) Descriptor() ([]byte, []int) { return file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP(), []int{0} } -func (x *SinkRequest) GetKeys() []string { +func (x *SinkRequest) GetRequest() *SinkRequest_Request { if x != nil { - return x.Keys + return x.Request } return nil } -func (x *SinkRequest) GetValue() []byte { +func (x *SinkRequest) GetStatus() *TransmissionStatus { if x != nil { - return x.Value + return x.Status } return nil } -func (x *SinkRequest) GetEventTime() *timestamppb.Timestamp { +func (x *SinkRequest) GetHandshake() *Handshake { if x != nil { - return x.EventTime + return x.Handshake } return nil } -func (x *SinkRequest) GetWatermark() *timestamppb.Timestamp { - if x != nil { - return x.Watermark +// Handshake message between client and server to indicate the start of transmission. +type Handshake struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Required field indicating the start of transmission. + Sot bool `protobuf:"varint,1,opt,name=sot,proto3" json:"sot,omitempty"` +} + +func (x *Handshake) Reset() { + *x = Handshake{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return nil } -func (x *SinkRequest) GetId() string { - if x != nil { - return x.Id +func (x *Handshake) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Handshake) ProtoMessage() {} + +func (x *Handshake) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return "" + return mi.MessageOf(x) } -func (x *SinkRequest) GetHeaders() map[string]string { +// Deprecated: Use Handshake.ProtoReflect.Descriptor instead. +func (*Handshake) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP(), []int{1} +} + +func (x *Handshake) GetSot() bool { if x != nil { - return x.Headers + return x.Sot } - return nil + return false } // * @@ -174,7 +203,7 @@ type ReadyResponse struct { func (x *ReadyResponse) Reset() { *x = ReadyResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[1] + mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -187,7 +216,7 @@ func (x *ReadyResponse) String() string { func (*ReadyResponse) ProtoMessage() {} func (x *ReadyResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[1] + mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -200,7 +229,7 @@ func (x *ReadyResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadyResponse.ProtoReflect.Descriptor instead. func (*ReadyResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP(), []int{1} + return file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP(), []int{2} } func (x *ReadyResponse) GetReady() bool { @@ -210,6 +239,55 @@ func (x *ReadyResponse) GetReady() bool { return false } +// * +// TransmissionStatus is the status of the transmission. +type TransmissionStatus struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Eot bool `protobuf:"varint,1,opt,name=eot,proto3" json:"eot,omitempty"` +} + +func (x *TransmissionStatus) Reset() { + *x = TransmissionStatus{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TransmissionStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TransmissionStatus) ProtoMessage() {} + +func (x *TransmissionStatus) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TransmissionStatus.ProtoReflect.Descriptor instead. +func (*TransmissionStatus) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP(), []int{3} +} + +func (x *TransmissionStatus) GetEot() bool { + if x != nil { + return x.Eot + } + return false +} + // * // SinkResponse is the individual response of each message written to the sink. type SinkResponse struct { @@ -217,13 +295,15 @@ type SinkResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Results []*SinkResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + Results []*SinkResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + Handshake *Handshake `protobuf:"bytes,2,opt,name=handshake,proto3,oneof" json:"handshake,omitempty"` + Status *TransmissionStatus `protobuf:"bytes,3,opt,name=status,proto3,oneof" json:"status,omitempty"` } func (x *SinkResponse) Reset() { *x = SinkResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[2] + mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -236,7 +316,7 @@ func (x *SinkResponse) String() string { func (*SinkResponse) ProtoMessage() {} func (x *SinkResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[2] + mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -249,7 +329,7 @@ func (x *SinkResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SinkResponse.ProtoReflect.Descriptor instead. func (*SinkResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP(), []int{2} + return file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP(), []int{4} } func (x *SinkResponse) GetResults() []*SinkResponse_Result { @@ -259,6 +339,107 @@ func (x *SinkResponse) GetResults() []*SinkResponse_Result { return nil } +func (x *SinkResponse) GetHandshake() *Handshake { + if x != nil { + return x.Handshake + } + return nil +} + +func (x *SinkResponse) GetStatus() *TransmissionStatus { + if x != nil { + return x.Status + } + return nil +} + +type SinkRequest_Request struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` + Id string `protobuf:"bytes,5,opt,name=id,proto3" json:"id,omitempty"` + Headers map[string]string `protobuf:"bytes,6,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *SinkRequest_Request) Reset() { + *x = SinkRequest_Request{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SinkRequest_Request) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SinkRequest_Request) ProtoMessage() {} + +func (x *SinkRequest_Request) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SinkRequest_Request.ProtoReflect.Descriptor instead. +func (*SinkRequest_Request) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP(), []int{0, 0} +} + +func (x *SinkRequest_Request) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + +func (x *SinkRequest_Request) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *SinkRequest_Request) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + +func (x *SinkRequest_Request) GetWatermark() *timestamppb.Timestamp { + if x != nil { + return x.Watermark + } + return nil +} + +func (x *SinkRequest_Request) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *SinkRequest_Request) GetHeaders() map[string]string { + if x != nil { + return x.Headers + } + return nil +} + type SinkResponse_Result struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -275,7 +456,7 @@ type SinkResponse_Result struct { func (x *SinkResponse_Result) Reset() { *x = SinkResponse_Result{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[4] + mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -288,7 +469,7 @@ func (x *SinkResponse_Result) String() string { func (*SinkResponse_Result) ProtoMessage() {} func (x *SinkResponse_Result) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[4] + mi := &file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -301,7 +482,7 @@ func (x *SinkResponse_Result) ProtoReflect() protoreflect.Message { // Deprecated: Use SinkResponse_Result.ProtoReflect.Descriptor instead. func (*SinkResponse_Result) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP(), []int{2, 0} + return file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP(), []int{4, 0} } func (x *SinkResponse_Result) GetId() string { @@ -334,8 +515,19 @@ var file_pkg_apis_proto_sink_v1_sink_proto_rawDesc = []byte{ 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xb5, 0x02, 0x0a, 0x0b, 0x53, - 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, + 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xfb, 0x03, 0x0a, 0x0b, 0x53, + 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x36, 0x0a, 0x07, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x69, + 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x33, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x54, 0x72, 0x61, + 0x6e, 0x73, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, + 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x35, 0x0a, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, + 0x68, 0x61, 0x6b, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x73, 0x69, 0x6e, + 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x48, 0x00, + 0x52, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x88, 0x01, 0x01, 0x1a, 0xb9, + 0x02, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, @@ -346,42 +538,57 @@ var file_pkg_apis_proto_sink_v1_sink_proto_rawDesc = []byte{ 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x3b, 0x0a, 0x07, 0x68, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x73, 0x69, 0x6e, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x43, 0x0a, 0x07, 0x68, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x22, 0xa2, 0x01, 0x0a, 0x0c, 0x53, 0x69, - 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x69, - 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x73, 0x1a, 0x5a, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x27, 0x0a, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x73, - 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x2a, 0x30, - 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, - 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, - 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x02, - 0x32, 0x7a, 0x0a, 0x04, 0x53, 0x69, 0x6e, 0x6b, 0x12, 0x37, 0x0a, 0x06, 0x53, 0x69, 0x6e, 0x6b, - 0x46, 0x6e, 0x12, 0x14, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x6e, - 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, - 0x76, 0x31, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, - 0x01, 0x12, 0x39, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x52, - 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x38, 0x5a, 0x36, - 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, - 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, - 0x69, 0x6e, 0x6b, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x1a, 0x3a, + 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x68, + 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x22, 0x1d, 0x0a, 0x09, 0x48, 0x61, 0x6e, 0x64, + 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x03, 0x73, 0x6f, 0x74, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, + 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x22, 0x26, + 0x0a, 0x12, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x03, 0x65, 0x6f, 0x74, 0x22, 0xac, 0x02, 0x0a, 0x0c, 0x53, 0x69, 0x6e, 0x6b, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, + 0x76, 0x31, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, + 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, + 0x35, 0x0a, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x6e, + 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x48, 0x00, 0x52, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, + 0x61, 0x6b, 0x65, 0x88, 0x01, 0x01, 0x12, 0x38, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, + 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x6d, 0x69, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x48, 0x01, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x88, 0x01, 0x01, + 0x1a, 0x5a, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x27, 0x0a, 0x06, 0x73, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x73, 0x69, 0x6e, + 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x42, 0x0c, 0x0a, 0x0a, + 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x2a, 0x30, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, + 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, + 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x46, 0x41, 0x4c, + 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x02, 0x32, 0x7c, 0x0a, 0x04, 0x53, 0x69, 0x6e, 0x6b, 0x12, + 0x39, 0x0a, 0x06, 0x53, 0x69, 0x6e, 0x6b, 0x46, 0x6e, 0x12, 0x14, 0x2e, 0x73, 0x69, 0x6e, 0x6b, + 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x15, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x39, 0x0a, 0x07, 0x49, 0x73, + 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, + 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x38, 0x5a, 0x36, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, + 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, + 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x69, 0x6e, 0x6b, 0x2f, 0x76, 0x31, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -397,32 +604,40 @@ func file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP() []byte { } var file_pkg_apis_proto_sink_v1_sink_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_pkg_apis_proto_sink_v1_sink_proto_msgTypes = make([]protoimpl.MessageInfo, 5) -var file_pkg_apis_proto_sink_v1_sink_proto_goTypes = []interface{}{ +var file_pkg_apis_proto_sink_v1_sink_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_pkg_apis_proto_sink_v1_sink_proto_goTypes = []any{ (Status)(0), // 0: sink.v1.Status (*SinkRequest)(nil), // 1: sink.v1.SinkRequest - (*ReadyResponse)(nil), // 2: sink.v1.ReadyResponse - (*SinkResponse)(nil), // 3: sink.v1.SinkResponse - nil, // 4: sink.v1.SinkRequest.HeadersEntry - (*SinkResponse_Result)(nil), // 5: sink.v1.SinkResponse.Result - (*timestamppb.Timestamp)(nil), // 6: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 7: google.protobuf.Empty + (*Handshake)(nil), // 2: sink.v1.Handshake + (*ReadyResponse)(nil), // 3: sink.v1.ReadyResponse + (*TransmissionStatus)(nil), // 4: sink.v1.TransmissionStatus + (*SinkResponse)(nil), // 5: sink.v1.SinkResponse + (*SinkRequest_Request)(nil), // 6: sink.v1.SinkRequest.Request + nil, // 7: sink.v1.SinkRequest.Request.HeadersEntry + (*SinkResponse_Result)(nil), // 8: sink.v1.SinkResponse.Result + (*timestamppb.Timestamp)(nil), // 9: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 10: google.protobuf.Empty } var file_pkg_apis_proto_sink_v1_sink_proto_depIdxs = []int32{ - 6, // 0: sink.v1.SinkRequest.event_time:type_name -> google.protobuf.Timestamp - 6, // 1: sink.v1.SinkRequest.watermark:type_name -> google.protobuf.Timestamp - 4, // 2: sink.v1.SinkRequest.headers:type_name -> sink.v1.SinkRequest.HeadersEntry - 5, // 3: sink.v1.SinkResponse.results:type_name -> sink.v1.SinkResponse.Result - 0, // 4: sink.v1.SinkResponse.Result.status:type_name -> sink.v1.Status - 1, // 5: sink.v1.Sink.SinkFn:input_type -> sink.v1.SinkRequest - 7, // 6: sink.v1.Sink.IsReady:input_type -> google.protobuf.Empty - 3, // 7: sink.v1.Sink.SinkFn:output_type -> sink.v1.SinkResponse - 2, // 8: sink.v1.Sink.IsReady:output_type -> sink.v1.ReadyResponse - 7, // [7:9] is the sub-list for method output_type - 5, // [5:7] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 6, // 0: sink.v1.SinkRequest.request:type_name -> sink.v1.SinkRequest.Request + 4, // 1: sink.v1.SinkRequest.status:type_name -> sink.v1.TransmissionStatus + 2, // 2: sink.v1.SinkRequest.handshake:type_name -> sink.v1.Handshake + 8, // 3: sink.v1.SinkResponse.results:type_name -> sink.v1.SinkResponse.Result + 2, // 4: sink.v1.SinkResponse.handshake:type_name -> sink.v1.Handshake + 4, // 5: sink.v1.SinkResponse.status:type_name -> sink.v1.TransmissionStatus + 9, // 6: sink.v1.SinkRequest.Request.event_time:type_name -> google.protobuf.Timestamp + 9, // 7: sink.v1.SinkRequest.Request.watermark:type_name -> google.protobuf.Timestamp + 7, // 8: sink.v1.SinkRequest.Request.headers:type_name -> sink.v1.SinkRequest.Request.HeadersEntry + 0, // 9: sink.v1.SinkResponse.Result.status:type_name -> sink.v1.Status + 1, // 10: sink.v1.Sink.SinkFn:input_type -> sink.v1.SinkRequest + 10, // 11: sink.v1.Sink.IsReady:input_type -> google.protobuf.Empty + 5, // 12: sink.v1.Sink.SinkFn:output_type -> sink.v1.SinkResponse + 3, // 13: sink.v1.Sink.IsReady:output_type -> sink.v1.ReadyResponse + 12, // [12:14] is the sub-list for method output_type + 10, // [10:12] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_pkg_apis_proto_sink_v1_sink_proto_init() } @@ -431,7 +646,7 @@ func file_pkg_apis_proto_sink_v1_sink_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*SinkRequest); i { case 0: return &v.state @@ -443,7 +658,19 @@ func file_pkg_apis_proto_sink_v1_sink_proto_init() { return nil } } - file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[1].Exporter = func(v any, i int) any { + switch v := v.(*Handshake); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*ReadyResponse); i { case 0: return &v.state @@ -455,7 +682,19 @@ func file_pkg_apis_proto_sink_v1_sink_proto_init() { return nil } } - file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[3].Exporter = func(v any, i int) any { + switch v := v.(*TransmissionStatus); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*SinkResponse); i { case 0: return &v.state @@ -467,7 +706,19 @@ func file_pkg_apis_proto_sink_v1_sink_proto_init() { return nil } } - file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[5].Exporter = func(v any, i int) any { + switch v := v.(*SinkRequest_Request); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[7].Exporter = func(v any, i int) any { switch v := v.(*SinkResponse_Result); i { case 0: return &v.state @@ -480,13 +731,15 @@ func file_pkg_apis_proto_sink_v1_sink_proto_init() { } } } + file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[0].OneofWrappers = []any{} + file_pkg_apis_proto_sink_v1_sink_proto_msgTypes[4].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_apis_proto_sink_v1_sink_proto_rawDesc, NumEnums: 1, - NumMessages: 5, + NumMessages: 8, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/apis/proto/sink/v1/sink.proto b/pkg/apis/proto/sink/v1/sink.proto index 0bdd5a27..a042fa4d 100644 --- a/pkg/apis/proto/sink/v1/sink.proto +++ b/pkg/apis/proto/sink/v1/sink.proto @@ -10,7 +10,7 @@ package sink.v1; service Sink { // SinkFn writes the request to a user defined sink. - rpc SinkFn(stream SinkRequest) returns (SinkResponse); + rpc SinkFn(stream SinkRequest) returns (stream SinkResponse); // IsReady is the heartbeat endpoint for gRPC. rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); @@ -20,12 +20,29 @@ service Sink { * SinkRequest represents a request element. */ message SinkRequest { - repeated string keys = 1; - bytes value = 2; - google.protobuf.Timestamp event_time = 3; - google.protobuf.Timestamp watermark = 4; - string id = 5; - map headers = 6; + message Request { + repeated string keys = 1; + bytes value = 2; + google.protobuf.Timestamp event_time = 3; + google.protobuf.Timestamp watermark = 4; + string id = 5; + map headers = 6; + } + // Required field indicating the request. + Request request = 1; + // Required field indicating the status of the request. + // If eot is set to true, it indicates the end of transmission. + TransmissionStatus status = 2; + // optional field indicating the handshake message. + optional Handshake handshake = 3; +} + +/* + * Handshake message between client and server to indicate the start of transmission. + */ +message Handshake { + // Required field indicating the start of transmission. + bool sot = 1; } /** @@ -35,6 +52,13 @@ message ReadyResponse { bool ready = 1; } +/** + * TransmissionStatus is the status of the transmission. + */ +message TransmissionStatus { + bool eot = 1; +} + /* * Status is the status of the response. */ @@ -57,4 +81,6 @@ message SinkResponse { string err_msg = 3; } repeated Result results = 1; + optional Handshake handshake = 2; + optional TransmissionStatus status = 3; } \ No newline at end of file diff --git a/pkg/apis/proto/sink/v1/sink_grpc.pb.go b/pkg/apis/proto/sink/v1/sink_grpc.pb.go index bc50da8e..7a975673 100644 --- a/pkg/apis/proto/sink/v1/sink_grpc.pb.go +++ b/pkg/apis/proto/sink/v1/sink_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v4.25.1 // source: pkg/apis/proto/sink/v1/sink.proto @@ -16,8 +16,13 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + Sink_SinkFn_FullMethodName = "/sink.v1.Sink/SinkFn" + Sink_IsReady_FullMethodName = "/sink.v1.Sink/IsReady" +) // SinkClient is the client API for Sink service. // @@ -38,17 +43,18 @@ func NewSinkClient(cc grpc.ClientConnInterface) SinkClient { } func (c *sinkClient) SinkFn(ctx context.Context, opts ...grpc.CallOption) (Sink_SinkFnClient, error) { - stream, err := c.cc.NewStream(ctx, &Sink_ServiceDesc.Streams[0], "/sink.v1.Sink/SinkFn", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Sink_ServiceDesc.Streams[0], Sink_SinkFn_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &sinkSinkFnClient{stream} + x := &sinkSinkFnClient{ClientStream: stream} return x, nil } type Sink_SinkFnClient interface { Send(*SinkRequest) error - CloseAndRecv() (*SinkResponse, error) + Recv() (*SinkResponse, error) grpc.ClientStream } @@ -60,10 +66,7 @@ func (x *sinkSinkFnClient) Send(m *SinkRequest) error { return x.ClientStream.SendMsg(m) } -func (x *sinkSinkFnClient) CloseAndRecv() (*SinkResponse, error) { - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } +func (x *sinkSinkFnClient) Recv() (*SinkResponse, error) { m := new(SinkResponse) if err := x.ClientStream.RecvMsg(m); err != nil { return nil, err @@ -72,8 +75,9 @@ func (x *sinkSinkFnClient) CloseAndRecv() (*SinkResponse, error) { } func (c *sinkClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ReadyResponse) - err := c.cc.Invoke(ctx, "/sink.v1.Sink/IsReady", in, out, opts...) + err := c.cc.Invoke(ctx, Sink_IsReady_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -115,11 +119,11 @@ func RegisterSinkServer(s grpc.ServiceRegistrar, srv SinkServer) { } func _Sink_SinkFn_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(SinkServer).SinkFn(&sinkSinkFnServer{stream}) + return srv.(SinkServer).SinkFn(&sinkSinkFnServer{ServerStream: stream}) } type Sink_SinkFnServer interface { - SendAndClose(*SinkResponse) error + Send(*SinkResponse) error Recv() (*SinkRequest, error) grpc.ServerStream } @@ -128,7 +132,7 @@ type sinkSinkFnServer struct { grpc.ServerStream } -func (x *sinkSinkFnServer) SendAndClose(m *SinkResponse) error { +func (x *sinkSinkFnServer) Send(m *SinkResponse) error { return x.ServerStream.SendMsg(m) } @@ -150,7 +154,7 @@ func _Sink_IsReady_Handler(srv interface{}, ctx context.Context, dec func(interf } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/sink.v1.Sink/IsReady", + FullMethod: Sink_IsReady_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(SinkServer).IsReady(ctx, req.(*emptypb.Empty)) @@ -174,6 +178,7 @@ var Sink_ServiceDesc = grpc.ServiceDesc{ { StreamName: "SinkFn", Handler: _Sink_SinkFn_Handler, + ServerStreams: true, ClientStreams: true, }, }, diff --git a/pkg/apis/proto/sink/v1/sinkmock/sinkmock.go b/pkg/apis/proto/sink/v1/sinkmock/sinkmock.go index 6ee22b6e..442fc9ca 100644 --- a/pkg/apis/proto/sink/v1/sinkmock/sinkmock.go +++ b/pkg/apis/proto/sink/v1/sinkmock/sinkmock.go @@ -101,21 +101,6 @@ func (m *MockSink_SinkFnClient) EXPECT() *MockSink_SinkFnClientMockRecorder { return m.recorder } -// CloseAndRecv mocks base method. -func (m *MockSink_SinkFnClient) CloseAndRecv() (*v1.SinkResponse, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CloseAndRecv") - ret0, _ := ret[0].(*v1.SinkResponse) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CloseAndRecv indicates an expected call of CloseAndRecv. -func (mr *MockSink_SinkFnClientMockRecorder) CloseAndRecv() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseAndRecv", reflect.TypeOf((*MockSink_SinkFnClient)(nil).CloseAndRecv)) -} - // CloseSend mocks base method. func (m *MockSink_SinkFnClient) CloseSend() error { m.ctrl.T.Helper() @@ -159,6 +144,21 @@ func (mr *MockSink_SinkFnClientMockRecorder) Header() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Header", reflect.TypeOf((*MockSink_SinkFnClient)(nil).Header)) } +// Recv mocks base method. +func (m *MockSink_SinkFnClient) Recv() (*v1.SinkResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Recv") + ret0, _ := ret[0].(*v1.SinkResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Recv indicates an expected call of Recv. +func (mr *MockSink_SinkFnClientMockRecorder) Recv() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Recv", reflect.TypeOf((*MockSink_SinkFnClient)(nil).Recv)) +} + // RecvMsg mocks base method. func (m *MockSink_SinkFnClient) RecvMsg(arg0 interface{}) error { m.ctrl.T.Helper() diff --git a/pkg/apis/proto/source/v1/mockgen.go b/pkg/apis/proto/source/v1/mockgen.go index 89dad73d..e2ea1193 100644 --- a/pkg/apis/proto/source/v1/mockgen.go +++ b/pkg/apis/proto/source/v1/mockgen.go @@ -1,3 +1,3 @@ package v1 -//go:generate mockgen -destination sourcemock/sourcemock.go -package sourcemock github.com/numaproj/numaflow-go/pkg/apis/proto/source/v1 SourceClient,Source_ReadFnClient +//go:generate mockgen -destination sourcemock/sourcemock.go -package sourcemock github.com/numaproj/numaflow-go/pkg/apis/proto/source/v1 SourceClient,Source_ReadFnClient,Source_AckFnClient diff --git a/pkg/apis/proto/source/v1/source.pb.go b/pkg/apis/proto/source/v1/source.pb.go index 7927e25e..c69b85fe 100644 --- a/pkg/apis/proto/source/v1/source.pb.go +++ b/pkg/apis/proto/source/v1/source.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.34.2 // protoc v4.25.1 // source: pkg/apis/proto/source/v1/source.proto @@ -22,6 +22,149 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// Code to indicate the status of the response. +type ReadResponse_Status_Code int32 + +const ( + ReadResponse_Status_SUCCESS ReadResponse_Status_Code = 0 + ReadResponse_Status_FAILURE ReadResponse_Status_Code = 1 +) + +// Enum value maps for ReadResponse_Status_Code. +var ( + ReadResponse_Status_Code_name = map[int32]string{ + 0: "SUCCESS", + 1: "FAILURE", + } + ReadResponse_Status_Code_value = map[string]int32{ + "SUCCESS": 0, + "FAILURE": 1, + } +) + +func (x ReadResponse_Status_Code) Enum() *ReadResponse_Status_Code { + p := new(ReadResponse_Status_Code) + *p = x + return p +} + +func (x ReadResponse_Status_Code) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ReadResponse_Status_Code) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_apis_proto_source_v1_source_proto_enumTypes[0].Descriptor() +} + +func (ReadResponse_Status_Code) Type() protoreflect.EnumType { + return &file_pkg_apis_proto_source_v1_source_proto_enumTypes[0] +} + +func (x ReadResponse_Status_Code) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ReadResponse_Status_Code.Descriptor instead. +func (ReadResponse_Status_Code) EnumDescriptor() ([]byte, []int) { + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{2, 1, 0} +} + +// Error to indicate the error type. If the code is FAILURE, then the error field will be populated. +type ReadResponse_Status_Error int32 + +const ( + ReadResponse_Status_UNACKED ReadResponse_Status_Error = 0 + ReadResponse_Status_OTHER ReadResponse_Status_Error = 1 +) + +// Enum value maps for ReadResponse_Status_Error. +var ( + ReadResponse_Status_Error_name = map[int32]string{ + 0: "UNACKED", + 1: "OTHER", + } + ReadResponse_Status_Error_value = map[string]int32{ + "UNACKED": 0, + "OTHER": 1, + } +) + +func (x ReadResponse_Status_Error) Enum() *ReadResponse_Status_Error { + p := new(ReadResponse_Status_Error) + *p = x + return p +} + +func (x ReadResponse_Status_Error) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ReadResponse_Status_Error) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_apis_proto_source_v1_source_proto_enumTypes[1].Descriptor() +} + +func (ReadResponse_Status_Error) Type() protoreflect.EnumType { + return &file_pkg_apis_proto_source_v1_source_proto_enumTypes[1] +} + +func (x ReadResponse_Status_Error) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ReadResponse_Status_Error.Descriptor instead. +func (ReadResponse_Status_Error) EnumDescriptor() ([]byte, []int) { + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{2, 1, 1} +} + +// Handshake message between client and server to indicate the start of transmission. +type Handshake struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Required field indicating the start of transmission. + Sot bool `protobuf:"varint,1,opt,name=sot,proto3" json:"sot,omitempty"` +} + +func (x *Handshake) Reset() { + *x = Handshake{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Handshake) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Handshake) ProtoMessage() {} + +func (x *Handshake) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Handshake.ProtoReflect.Descriptor instead. +func (*Handshake) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{0} +} + +func (x *Handshake) GetSot() bool { + if x != nil { + return x.Sot + } + return false +} + // ReadRequest is the request for reading datum stream from user defined source. type ReadRequest struct { state protoimpl.MessageState @@ -29,13 +172,14 @@ type ReadRequest struct { unknownFields protoimpl.UnknownFields // Required field indicating the request. - Request *ReadRequest_Request `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` + Request *ReadRequest_Request `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` + Handshake *Handshake `protobuf:"bytes,2,opt,name=handshake,proto3,oneof" json:"handshake,omitempty"` } func (x *ReadRequest) Reset() { *x = ReadRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[0] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -48,7 +192,7 @@ func (x *ReadRequest) String() string { func (*ReadRequest) ProtoMessage() {} func (x *ReadRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[0] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[1] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -61,7 +205,7 @@ func (x *ReadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadRequest.ProtoReflect.Descriptor instead. func (*ReadRequest) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{0} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{1} } func (x *ReadRequest) GetRequest() *ReadRequest_Request { @@ -71,6 +215,13 @@ func (x *ReadRequest) GetRequest() *ReadRequest_Request { return nil } +func (x *ReadRequest) GetHandshake() *Handshake { + if x != nil { + return x.Handshake + } + return nil +} + // ReadResponse is the response for reading datum stream from user defined source. type ReadResponse struct { state protoimpl.MessageState @@ -79,12 +230,16 @@ type ReadResponse struct { // Required field holding the result. Result *ReadResponse_Result `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` + // Status of the response. Holds the end of transmission flag and the status code. + Status *ReadResponse_Status `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"` + // Handshake message between client and server to indicate the start of transmission. + Handshake *Handshake `protobuf:"bytes,3,opt,name=handshake,proto3,oneof" json:"handshake,omitempty"` } func (x *ReadResponse) Reset() { *x = ReadResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[1] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -97,7 +252,7 @@ func (x *ReadResponse) String() string { func (*ReadResponse) ProtoMessage() {} func (x *ReadResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[1] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -110,7 +265,7 @@ func (x *ReadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadResponse.ProtoReflect.Descriptor instead. func (*ReadResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{1} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{2} } func (x *ReadResponse) GetResult() *ReadResponse_Result { @@ -120,6 +275,20 @@ func (x *ReadResponse) GetResult() *ReadResponse_Result { return nil } +func (x *ReadResponse) GetStatus() *ReadResponse_Status { + if x != nil { + return x.Status + } + return nil +} + +func (x *ReadResponse) GetHandshake() *Handshake { + if x != nil { + return x.Handshake + } + return nil +} + // AckRequest is the request for acknowledging datum. // It takes a list of offsets to be acknowledged. type AckRequest struct { @@ -128,13 +297,14 @@ type AckRequest struct { unknownFields protoimpl.UnknownFields // Required field holding the request. The list will be ordered and will have the same order as the original Read response. - Request *AckRequest_Request `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` + Request *AckRequest_Request `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` + Handshake *Handshake `protobuf:"bytes,2,opt,name=handshake,proto3,oneof" json:"handshake,omitempty"` } func (x *AckRequest) Reset() { *x = AckRequest{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[2] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -147,7 +317,7 @@ func (x *AckRequest) String() string { func (*AckRequest) ProtoMessage() {} func (x *AckRequest) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[2] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -160,7 +330,7 @@ func (x *AckRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AckRequest.ProtoReflect.Descriptor instead. func (*AckRequest) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{2} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{3} } func (x *AckRequest) GetRequest() *AckRequest_Request { @@ -170,6 +340,13 @@ func (x *AckRequest) GetRequest() *AckRequest_Request { return nil } +func (x *AckRequest) GetHandshake() *Handshake { + if x != nil { + return x.Handshake + } + return nil +} + // AckResponse is the response for acknowledging datum. It contains one empty field confirming // the batch of offsets that have been successfully acknowledged. The contract between client and server // is that the server will only return the AckResponse if the ack request is successful. @@ -185,12 +362,14 @@ type AckResponse struct { // Required field holding the result. Result *AckResponse_Result `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` + // Handshake message between client and server to indicate the start of transmission. + Handshake *Handshake `protobuf:"bytes,2,opt,name=handshake,proto3,oneof" json:"handshake,omitempty"` } func (x *AckResponse) Reset() { *x = AckResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[3] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -203,7 +382,7 @@ func (x *AckResponse) String() string { func (*AckResponse) ProtoMessage() {} func (x *AckResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[3] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[4] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -216,7 +395,7 @@ func (x *AckResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AckResponse.ProtoReflect.Descriptor instead. func (*AckResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{3} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{4} } func (x *AckResponse) GetResult() *AckResponse_Result { @@ -226,6 +405,13 @@ func (x *AckResponse) GetResult() *AckResponse_Result { return nil } +func (x *AckResponse) GetHandshake() *Handshake { + if x != nil { + return x.Handshake + } + return nil +} + // ReadyResponse is the health check result for user defined source. type ReadyResponse struct { state protoimpl.MessageState @@ -239,7 +425,7 @@ type ReadyResponse struct { func (x *ReadyResponse) Reset() { *x = ReadyResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[4] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -252,7 +438,7 @@ func (x *ReadyResponse) String() string { func (*ReadyResponse) ProtoMessage() {} func (x *ReadyResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[4] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -265,7 +451,7 @@ func (x *ReadyResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadyResponse.ProtoReflect.Descriptor instead. func (*ReadyResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{4} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{5} } func (x *ReadyResponse) GetReady() bool { @@ -288,7 +474,7 @@ type PendingResponse struct { func (x *PendingResponse) Reset() { *x = PendingResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[5] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -301,7 +487,7 @@ func (x *PendingResponse) String() string { func (*PendingResponse) ProtoMessage() {} func (x *PendingResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[5] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -314,7 +500,7 @@ func (x *PendingResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PendingResponse.ProtoReflect.Descriptor instead. func (*PendingResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{5} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{6} } func (x *PendingResponse) GetResult() *PendingResponse_Result { @@ -337,7 +523,7 @@ type PartitionsResponse struct { func (x *PartitionsResponse) Reset() { *x = PartitionsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[6] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -350,7 +536,7 @@ func (x *PartitionsResponse) String() string { func (*PartitionsResponse) ProtoMessage() {} func (x *PartitionsResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[6] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[7] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -363,7 +549,7 @@ func (x *PartitionsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use PartitionsResponse.ProtoReflect.Descriptor instead. func (*PartitionsResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{6} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{7} } func (x *PartitionsResponse) GetResult() *PartitionsResponse_Result { @@ -393,7 +579,7 @@ type Offset struct { func (x *Offset) Reset() { *x = Offset{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[7] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -406,7 +592,7 @@ func (x *Offset) String() string { func (*Offset) ProtoMessage() {} func (x *Offset) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[7] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[8] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -419,7 +605,7 @@ func (x *Offset) ProtoReflect() protoreflect.Message { // Deprecated: Use Offset.ProtoReflect.Descriptor instead. func (*Offset) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{7} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{8} } func (x *Offset) GetOffset() []byte { @@ -452,7 +638,7 @@ type ReadRequest_Request struct { func (x *ReadRequest_Request) Reset() { *x = ReadRequest_Request{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[8] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -465,7 +651,7 @@ func (x *ReadRequest_Request) String() string { func (*ReadRequest_Request) ProtoMessage() {} func (x *ReadRequest_Request) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[8] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[9] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -478,7 +664,7 @@ func (x *ReadRequest_Request) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadRequest_Request.ProtoReflect.Descriptor instead. func (*ReadRequest_Request) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{0, 0} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{1, 0} } func (x *ReadRequest_Request) GetNumRecords() uint64 { @@ -520,7 +706,7 @@ type ReadResponse_Result struct { func (x *ReadResponse_Result) Reset() { *x = ReadResponse_Result{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[9] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -533,7 +719,7 @@ func (x *ReadResponse_Result) String() string { func (*ReadResponse_Result) ProtoMessage() {} func (x *ReadResponse_Result) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[9] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[10] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -546,7 +732,7 @@ func (x *ReadResponse_Result) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadResponse_Result.ProtoReflect.Descriptor instead. func (*ReadResponse_Result) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{1, 0} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{2, 0} } func (x *ReadResponse_Result) GetPayload() []byte { @@ -584,22 +770,91 @@ func (x *ReadResponse_Result) GetHeaders() map[string]string { return nil } +type ReadResponse_Status struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // End of transmission flag. + Eot bool `protobuf:"varint,1,opt,name=eot,proto3" json:"eot,omitempty"` + Code ReadResponse_Status_Code `protobuf:"varint,2,opt,name=code,proto3,enum=source.v1.ReadResponse_Status_Code" json:"code,omitempty"` + Error *ReadResponse_Status_Error `protobuf:"varint,3,opt,name=error,proto3,enum=source.v1.ReadResponse_Status_Error,oneof" json:"error,omitempty"` + Msg *string `protobuf:"bytes,4,opt,name=msg,proto3,oneof" json:"msg,omitempty"` +} + +func (x *ReadResponse_Status) Reset() { + *x = ReadResponse_Status{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReadResponse_Status) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReadResponse_Status) ProtoMessage() {} + +func (x *ReadResponse_Status) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReadResponse_Status.ProtoReflect.Descriptor instead. +func (*ReadResponse_Status) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{2, 1} +} + +func (x *ReadResponse_Status) GetEot() bool { + if x != nil { + return x.Eot + } + return false +} + +func (x *ReadResponse_Status) GetCode() ReadResponse_Status_Code { + if x != nil { + return x.Code + } + return ReadResponse_Status_SUCCESS +} + +func (x *ReadResponse_Status) GetError() ReadResponse_Status_Error { + if x != nil && x.Error != nil { + return *x.Error + } + return ReadResponse_Status_UNACKED +} + +func (x *ReadResponse_Status) GetMsg() string { + if x != nil && x.Msg != nil { + return *x.Msg + } + return "" +} + type AckRequest_Request struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // Required field holding a list of offsets to be acknowledged. - // The offsets must be strictly corresponding to the previously read batch, - // meaning the offsets must be in the same order as the datum responses in the ReadResponse. - // By enforcing ordering, we can save deserialization effort on the server side, assuming the server keeps a local copy of the raw/un-serialized offsets. + // Required field holding the offset to be acked Offsets []*Offset `protobuf:"bytes,1,rep,name=offsets,proto3" json:"offsets,omitempty"` } func (x *AckRequest_Request) Reset() { *x = AckRequest_Request{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[11] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -612,7 +867,7 @@ func (x *AckRequest_Request) String() string { func (*AckRequest_Request) ProtoMessage() {} func (x *AckRequest_Request) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[11] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -625,7 +880,7 @@ func (x *AckRequest_Request) ProtoReflect() protoreflect.Message { // Deprecated: Use AckRequest_Request.ProtoReflect.Descriptor instead. func (*AckRequest_Request) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{2, 0} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{3, 0} } func (x *AckRequest_Request) GetOffsets() []*Offset { @@ -647,7 +902,7 @@ type AckResponse_Result struct { func (x *AckResponse_Result) Reset() { *x = AckResponse_Result{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[12] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -660,7 +915,7 @@ func (x *AckResponse_Result) String() string { func (*AckResponse_Result) ProtoMessage() {} func (x *AckResponse_Result) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[12] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -673,7 +928,7 @@ func (x *AckResponse_Result) ProtoReflect() protoreflect.Message { // Deprecated: Use AckResponse_Result.ProtoReflect.Descriptor instead. func (*AckResponse_Result) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{3, 0} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{4, 0} } func (x *AckResponse_Result) GetSuccess() *emptypb.Empty { @@ -696,7 +951,7 @@ type PendingResponse_Result struct { func (x *PendingResponse_Result) Reset() { *x = PendingResponse_Result{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[13] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -709,7 +964,7 @@ func (x *PendingResponse_Result) String() string { func (*PendingResponse_Result) ProtoMessage() {} func (x *PendingResponse_Result) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[13] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -722,7 +977,7 @@ func (x *PendingResponse_Result) ProtoReflect() protoreflect.Message { // Deprecated: Use PendingResponse_Result.ProtoReflect.Descriptor instead. func (*PendingResponse_Result) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{5, 0} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{6, 0} } func (x *PendingResponse_Result) GetCount() int64 { @@ -744,7 +999,7 @@ type PartitionsResponse_Result struct { func (x *PartitionsResponse_Result) Reset() { *x = PartitionsResponse_Result{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[14] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -757,7 +1012,7 @@ func (x *PartitionsResponse_Result) String() string { func (*PartitionsResponse_Result) ProtoMessage() {} func (x *PartitionsResponse_Result) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[14] + mi := &file_pkg_apis_proto_source_v1_source_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -770,7 +1025,7 @@ func (x *PartitionsResponse_Result) ProtoReflect() protoreflect.Message { // Deprecated: Use PartitionsResponse_Result.ProtoReflect.Descriptor instead. func (*PartitionsResponse_Result) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{6, 0} + return file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP(), []int{7, 0} } func (x *PartitionsResponse_Result) GetPartitions() []int32 { @@ -790,101 +1045,141 @@ var file_pkg_apis_proto_source_v1_source_proto_rawDesc = []byte{ 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x22, 0x97, 0x01, 0x0a, 0x0b, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x38, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x22, 0x1d, 0x0a, 0x09, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x12, 0x10, 0x0a, + 0x03, 0x73, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x73, 0x6f, 0x74, 0x22, + 0xde, 0x01, 0x0a, 0x0b, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x38, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1e, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, + 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x09, 0x68, 0x61, 0x6e, + 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, + 0x6b, 0x65, 0x48, 0x00, 0x52, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x88, + 0x01, 0x01, 0x1a, 0x4e, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, + 0x0b, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0a, 0x6e, 0x75, 0x6d, 0x52, 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x12, 0x22, + 0x0a, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x5f, 0x69, 0x6e, 0x5f, 0x6d, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x49, 0x6e, + 0x4d, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, + 0x22, 0xea, 0x05, 0x0a, 0x0c, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x36, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x4e, 0x0a, 0x07, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6e, 0x75, 0x6d, 0x5f, 0x72, 0x65, 0x63, - 0x6f, 0x72, 0x64, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0a, 0x6e, 0x75, 0x6d, 0x52, - 0x65, 0x63, 0x6f, 0x72, 0x64, 0x73, 0x12, 0x22, 0x0a, 0x0d, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, - 0x74, 0x5f, 0x69, 0x6e, 0x5f, 0x6d, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x74, - 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x49, 0x6e, 0x4d, 0x73, 0x22, 0xe8, 0x02, 0x0a, 0x0c, 0x52, - 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x06, 0x72, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x1a, 0x9f, 0x02, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x18, - 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x29, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, - 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x52, 0x06, 0x6f, 0x66, 0x66, - 0x73, 0x65, 0x74, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x12, - 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, - 0x79, 0x73, 0x12, 0x45, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, - 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, - 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, - 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x7d, 0x0a, 0x0a, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x36, 0x0a, 0x07, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x07, 0x6f, 0x66, 0x66, 0x73, 0x65, - 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x52, 0x07, 0x6f, 0x66, 0x66, - 0x73, 0x65, 0x74, 0x73, 0x22, 0x80, 0x01, 0x0a, 0x0b, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, + 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x36, 0x0a, 0x06, 0x73, 0x74, 0x61, + 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x12, 0x37, 0x0a, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, + 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x48, 0x00, 0x52, 0x09, 0x68, 0x61, + 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x88, 0x01, 0x01, 0x1a, 0x9f, 0x02, 0x0a, 0x06, 0x52, + 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x70, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, + 0x29, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x11, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x66, 0x66, 0x73, + 0x65, 0x74, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, + 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, + 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x45, 0x0a, 0x07, 0x68, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2b, 0x2e, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, + 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x80, 0x02, 0x0a, + 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6f, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x03, 0x65, 0x6f, 0x74, 0x12, 0x37, 0x0a, 0x04, 0x63, 0x6f, 0x64, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x2e, 0x43, 0x6f, 0x64, 0x65, 0x52, 0x04, 0x63, 0x6f, + 0x64, 0x65, 0x12, 0x3f, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x24, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, + 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, + 0x73, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, + 0x88, 0x01, 0x01, 0x12, 0x15, 0x0a, 0x03, 0x6d, 0x73, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x48, 0x01, 0x52, 0x03, 0x6d, 0x73, 0x67, 0x88, 0x01, 0x01, 0x22, 0x20, 0x0a, 0x04, 0x43, 0x6f, + 0x64, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, + 0x0b, 0x0a, 0x07, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, 0x22, 0x1f, 0x0a, 0x05, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x41, 0x43, 0x4b, 0x45, 0x44, + 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x4f, 0x54, 0x48, 0x45, 0x52, 0x10, 0x01, 0x42, 0x08, 0x0a, + 0x06, 0x5f, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, 0x06, 0x0a, 0x04, 0x5f, 0x6d, 0x73, 0x67, 0x42, + 0x0c, 0x0a, 0x0a, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x22, 0xc4, 0x01, + 0x0a, 0x0a, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x07, + 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x37, 0x0a, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, + 0x6b, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x48, 0x00, + 0x52, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x88, 0x01, 0x01, 0x1a, 0x36, + 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x07, 0x6f, 0x66, 0x66, + 0x73, 0x65, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x73, 0x6f, 0x75, + 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x52, 0x07, 0x6f, + 0x66, 0x66, 0x73, 0x65, 0x74, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x73, + 0x68, 0x61, 0x6b, 0x65, 0x22, 0xc7, 0x01, 0x0a, 0x0b, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x35, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x1a, 0x3a, 0x0a, 0x06, 0x52, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x30, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x52, 0x07, - 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x22, 0x6c, - 0x0a, 0x0f, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x39, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x21, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x65, - 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x1a, 0x1e, 0x0a, 0x06, - 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x7c, 0x0a, 0x12, - 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, - 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x1a, 0x28, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x70, 0x61, - 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0a, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x43, 0x0a, 0x06, 0x4f, 0x66, - 0x66, 0x73, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, - 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x32, - 0xc2, 0x02, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x12, 0x3b, 0x0a, 0x06, 0x52, 0x65, - 0x61, 0x64, 0x46, 0x6e, 0x12, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x36, 0x0a, 0x05, 0x41, 0x63, 0x6b, 0x46, 0x6e, - 0x12, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x6b, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x3f, 0x0a, 0x09, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x46, 0x6e, 0x12, 0x16, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1a, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, - 0x2e, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x45, 0x0a, 0x0c, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x46, 0x6e, - 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, - 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x73, 0x6f, 0x75, - 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, - 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, - 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2f, 0x76, 0x31, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x37, 0x0a, 0x09, 0x68, + 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, + 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x6e, 0x64, 0x73, + 0x68, 0x61, 0x6b, 0x65, 0x48, 0x00, 0x52, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, + 0x65, 0x88, 0x01, 0x01, 0x1a, 0x3a, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x30, + 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x22, 0x25, + 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, + 0x72, 0x65, 0x61, 0x64, 0x79, 0x22, 0x6c, 0x0a, 0x0f, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x06, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x1a, 0x1e, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x14, 0x0a, + 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x22, 0x7c, 0x0a, 0x12, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x06, 0x72, 0x65, 0x73, + 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, + 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x1a, 0x28, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, + 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x05, 0x52, 0x0a, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x22, 0x43, 0x0a, 0x06, 0x4f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6f, + 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x6f, 0x66, 0x66, + 0x73, 0x65, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x70, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x70, 0x61, 0x72, 0x74, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x32, 0xc8, 0x02, 0x0a, 0x06, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x12, 0x3d, 0x0a, 0x06, 0x52, 0x65, 0x61, 0x64, 0x46, 0x6e, 0x12, 0x16, 0x2e, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, + 0x52, 0x65, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, + 0x12, 0x3a, 0x0a, 0x05, 0x41, 0x63, 0x6b, 0x46, 0x6e, 0x12, 0x15, 0x2e, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x16, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x41, 0x63, 0x6b, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x3f, 0x0a, 0x09, + 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x46, 0x6e, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x1a, 0x1a, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, 0x31, 0x2e, 0x50, 0x65, + 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, + 0x0c, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x46, 0x6e, 0x12, 0x16, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x1d, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2e, 0x76, + 0x31, 0x2e, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, + 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x18, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x42, 0x3a, 0x5a, 0x38, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, + 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -899,53 +1194,65 @@ func file_pkg_apis_proto_source_v1_source_proto_rawDescGZIP() []byte { return file_pkg_apis_proto_source_v1_source_proto_rawDescData } -var file_pkg_apis_proto_source_v1_source_proto_msgTypes = make([]protoimpl.MessageInfo, 15) -var file_pkg_apis_proto_source_v1_source_proto_goTypes = []interface{}{ - (*ReadRequest)(nil), // 0: source.v1.ReadRequest - (*ReadResponse)(nil), // 1: source.v1.ReadResponse - (*AckRequest)(nil), // 2: source.v1.AckRequest - (*AckResponse)(nil), // 3: source.v1.AckResponse - (*ReadyResponse)(nil), // 4: source.v1.ReadyResponse - (*PendingResponse)(nil), // 5: source.v1.PendingResponse - (*PartitionsResponse)(nil), // 6: source.v1.PartitionsResponse - (*Offset)(nil), // 7: source.v1.Offset - (*ReadRequest_Request)(nil), // 8: source.v1.ReadRequest.Request - (*ReadResponse_Result)(nil), // 9: source.v1.ReadResponse.Result - nil, // 10: source.v1.ReadResponse.Result.HeadersEntry - (*AckRequest_Request)(nil), // 11: source.v1.AckRequest.Request - (*AckResponse_Result)(nil), // 12: source.v1.AckResponse.Result - (*PendingResponse_Result)(nil), // 13: source.v1.PendingResponse.Result - (*PartitionsResponse_Result)(nil), // 14: source.v1.PartitionsResponse.Result - (*timestamppb.Timestamp)(nil), // 15: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 16: google.protobuf.Empty +var file_pkg_apis_proto_source_v1_source_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_pkg_apis_proto_source_v1_source_proto_msgTypes = make([]protoimpl.MessageInfo, 17) +var file_pkg_apis_proto_source_v1_source_proto_goTypes = []any{ + (ReadResponse_Status_Code)(0), // 0: source.v1.ReadResponse.Status.Code + (ReadResponse_Status_Error)(0), // 1: source.v1.ReadResponse.Status.Error + (*Handshake)(nil), // 2: source.v1.Handshake + (*ReadRequest)(nil), // 3: source.v1.ReadRequest + (*ReadResponse)(nil), // 4: source.v1.ReadResponse + (*AckRequest)(nil), // 5: source.v1.AckRequest + (*AckResponse)(nil), // 6: source.v1.AckResponse + (*ReadyResponse)(nil), // 7: source.v1.ReadyResponse + (*PendingResponse)(nil), // 8: source.v1.PendingResponse + (*PartitionsResponse)(nil), // 9: source.v1.PartitionsResponse + (*Offset)(nil), // 10: source.v1.Offset + (*ReadRequest_Request)(nil), // 11: source.v1.ReadRequest.Request + (*ReadResponse_Result)(nil), // 12: source.v1.ReadResponse.Result + (*ReadResponse_Status)(nil), // 13: source.v1.ReadResponse.Status + nil, // 14: source.v1.ReadResponse.Result.HeadersEntry + (*AckRequest_Request)(nil), // 15: source.v1.AckRequest.Request + (*AckResponse_Result)(nil), // 16: source.v1.AckResponse.Result + (*PendingResponse_Result)(nil), // 17: source.v1.PendingResponse.Result + (*PartitionsResponse_Result)(nil), // 18: source.v1.PartitionsResponse.Result + (*timestamppb.Timestamp)(nil), // 19: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 20: google.protobuf.Empty } var file_pkg_apis_proto_source_v1_source_proto_depIdxs = []int32{ - 8, // 0: source.v1.ReadRequest.request:type_name -> source.v1.ReadRequest.Request - 9, // 1: source.v1.ReadResponse.result:type_name -> source.v1.ReadResponse.Result - 11, // 2: source.v1.AckRequest.request:type_name -> source.v1.AckRequest.Request - 12, // 3: source.v1.AckResponse.result:type_name -> source.v1.AckResponse.Result - 13, // 4: source.v1.PendingResponse.result:type_name -> source.v1.PendingResponse.Result - 14, // 5: source.v1.PartitionsResponse.result:type_name -> source.v1.PartitionsResponse.Result - 7, // 6: source.v1.ReadResponse.Result.offset:type_name -> source.v1.Offset - 15, // 7: source.v1.ReadResponse.Result.event_time:type_name -> google.protobuf.Timestamp - 10, // 8: source.v1.ReadResponse.Result.headers:type_name -> source.v1.ReadResponse.Result.HeadersEntry - 7, // 9: source.v1.AckRequest.Request.offsets:type_name -> source.v1.Offset - 16, // 10: source.v1.AckResponse.Result.success:type_name -> google.protobuf.Empty - 0, // 11: source.v1.Source.ReadFn:input_type -> source.v1.ReadRequest - 2, // 12: source.v1.Source.AckFn:input_type -> source.v1.AckRequest - 16, // 13: source.v1.Source.PendingFn:input_type -> google.protobuf.Empty - 16, // 14: source.v1.Source.PartitionsFn:input_type -> google.protobuf.Empty - 16, // 15: source.v1.Source.IsReady:input_type -> google.protobuf.Empty - 1, // 16: source.v1.Source.ReadFn:output_type -> source.v1.ReadResponse - 3, // 17: source.v1.Source.AckFn:output_type -> source.v1.AckResponse - 5, // 18: source.v1.Source.PendingFn:output_type -> source.v1.PendingResponse - 6, // 19: source.v1.Source.PartitionsFn:output_type -> source.v1.PartitionsResponse - 4, // 20: source.v1.Source.IsReady:output_type -> source.v1.ReadyResponse - 16, // [16:21] is the sub-list for method output_type - 11, // [11:16] is the sub-list for method input_type - 11, // [11:11] is the sub-list for extension type_name - 11, // [11:11] is the sub-list for extension extendee - 0, // [0:11] is the sub-list for field type_name + 11, // 0: source.v1.ReadRequest.request:type_name -> source.v1.ReadRequest.Request + 2, // 1: source.v1.ReadRequest.handshake:type_name -> source.v1.Handshake + 12, // 2: source.v1.ReadResponse.result:type_name -> source.v1.ReadResponse.Result + 13, // 3: source.v1.ReadResponse.status:type_name -> source.v1.ReadResponse.Status + 2, // 4: source.v1.ReadResponse.handshake:type_name -> source.v1.Handshake + 15, // 5: source.v1.AckRequest.request:type_name -> source.v1.AckRequest.Request + 2, // 6: source.v1.AckRequest.handshake:type_name -> source.v1.Handshake + 16, // 7: source.v1.AckResponse.result:type_name -> source.v1.AckResponse.Result + 2, // 8: source.v1.AckResponse.handshake:type_name -> source.v1.Handshake + 17, // 9: source.v1.PendingResponse.result:type_name -> source.v1.PendingResponse.Result + 18, // 10: source.v1.PartitionsResponse.result:type_name -> source.v1.PartitionsResponse.Result + 10, // 11: source.v1.ReadResponse.Result.offset:type_name -> source.v1.Offset + 19, // 12: source.v1.ReadResponse.Result.event_time:type_name -> google.protobuf.Timestamp + 14, // 13: source.v1.ReadResponse.Result.headers:type_name -> source.v1.ReadResponse.Result.HeadersEntry + 0, // 14: source.v1.ReadResponse.Status.code:type_name -> source.v1.ReadResponse.Status.Code + 1, // 15: source.v1.ReadResponse.Status.error:type_name -> source.v1.ReadResponse.Status.Error + 10, // 16: source.v1.AckRequest.Request.offsets:type_name -> source.v1.Offset + 20, // 17: source.v1.AckResponse.Result.success:type_name -> google.protobuf.Empty + 3, // 18: source.v1.Source.ReadFn:input_type -> source.v1.ReadRequest + 5, // 19: source.v1.Source.AckFn:input_type -> source.v1.AckRequest + 20, // 20: source.v1.Source.PendingFn:input_type -> google.protobuf.Empty + 20, // 21: source.v1.Source.PartitionsFn:input_type -> google.protobuf.Empty + 20, // 22: source.v1.Source.IsReady:input_type -> google.protobuf.Empty + 4, // 23: source.v1.Source.ReadFn:output_type -> source.v1.ReadResponse + 6, // 24: source.v1.Source.AckFn:output_type -> source.v1.AckResponse + 8, // 25: source.v1.Source.PendingFn:output_type -> source.v1.PendingResponse + 9, // 26: source.v1.Source.PartitionsFn:output_type -> source.v1.PartitionsResponse + 7, // 27: source.v1.Source.IsReady:output_type -> source.v1.ReadyResponse + 23, // [23:28] is the sub-list for method output_type + 18, // [18:23] is the sub-list for method input_type + 18, // [18:18] is the sub-list for extension type_name + 18, // [18:18] is the sub-list for extension extendee + 0, // [0:18] is the sub-list for field type_name } func init() { file_pkg_apis_proto_source_v1_source_proto_init() } @@ -954,7 +1261,19 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_pkg_apis_proto_source_v1_source_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[0].Exporter = func(v any, i int) any { + switch v := v.(*Handshake); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_source_v1_source_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*ReadRequest); i { case 0: return &v.state @@ -966,7 +1285,7 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*ReadResponse); i { case 0: return &v.state @@ -978,7 +1297,7 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*AckRequest); i { case 0: return &v.state @@ -990,7 +1309,7 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*AckResponse); i { case 0: return &v.state @@ -1002,7 +1321,7 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*ReadyResponse); i { case 0: return &v.state @@ -1014,7 +1333,7 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[6].Exporter = func(v any, i int) any { switch v := v.(*PendingResponse); i { case 0: return &v.state @@ -1026,7 +1345,7 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[7].Exporter = func(v any, i int) any { switch v := v.(*PartitionsResponse); i { case 0: return &v.state @@ -1038,7 +1357,7 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[8].Exporter = func(v any, i int) any { switch v := v.(*Offset); i { case 0: return &v.state @@ -1050,7 +1369,7 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[9].Exporter = func(v any, i int) any { switch v := v.(*ReadRequest_Request); i { case 0: return &v.state @@ -1062,7 +1381,7 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[10].Exporter = func(v any, i int) any { switch v := v.(*ReadResponse_Result); i { case 0: return &v.state @@ -1074,7 +1393,19 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[11].Exporter = func(v any, i int) any { + switch v := v.(*ReadResponse_Status); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_source_v1_source_proto_msgTypes[13].Exporter = func(v any, i int) any { switch v := v.(*AckRequest_Request); i { case 0: return &v.state @@ -1086,7 +1417,7 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[14].Exporter = func(v any, i int) any { switch v := v.(*AckResponse_Result); i { case 0: return &v.state @@ -1098,7 +1429,7 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[15].Exporter = func(v any, i int) any { switch v := v.(*PendingResponse_Result); i { case 0: return &v.state @@ -1110,7 +1441,7 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { return nil } } - file_pkg_apis_proto_source_v1_source_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_source_v1_source_proto_msgTypes[16].Exporter = func(v any, i int) any { switch v := v.(*PartitionsResponse_Result); i { case 0: return &v.state @@ -1123,18 +1454,24 @@ func file_pkg_apis_proto_source_v1_source_proto_init() { } } } + file_pkg_apis_proto_source_v1_source_proto_msgTypes[1].OneofWrappers = []any{} + file_pkg_apis_proto_source_v1_source_proto_msgTypes[2].OneofWrappers = []any{} + file_pkg_apis_proto_source_v1_source_proto_msgTypes[3].OneofWrappers = []any{} + file_pkg_apis_proto_source_v1_source_proto_msgTypes[4].OneofWrappers = []any{} + file_pkg_apis_proto_source_v1_source_proto_msgTypes[11].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_apis_proto_source_v1_source_proto_rawDesc, - NumEnums: 0, - NumMessages: 15, + NumEnums: 2, + NumMessages: 17, NumExtensions: 0, NumServices: 1, }, GoTypes: file_pkg_apis_proto_source_v1_source_proto_goTypes, DependencyIndexes: file_pkg_apis_proto_source_v1_source_proto_depIdxs, + EnumInfos: file_pkg_apis_proto_source_v1_source_proto_enumTypes, MessageInfos: file_pkg_apis_proto_source_v1_source_proto_msgTypes, }.Build() File_pkg_apis_proto_source_v1_source_proto = out.File diff --git a/pkg/apis/proto/source/v1/source.proto b/pkg/apis/proto/source/v1/source.proto index 9ee6f688..8cc8939b 100644 --- a/pkg/apis/proto/source/v1/source.proto +++ b/pkg/apis/proto/source/v1/source.proto @@ -9,16 +9,19 @@ package source.v1; service Source { // Read returns a stream of datum responses. - // The size of the returned ReadResponse is less than or equal to the num_records specified in ReadRequest. - // If the request timeout is reached on server side, the returned ReadResponse will contain all the datum that have been read (which could be an empty list). - rpc ReadFn(ReadRequest) returns (stream ReadResponse); + // The size of the returned responses is less than or equal to the num_records specified in each ReadRequest. + // If the request timeout is reached on the server side, the returned responses will contain all the datum that have been read (which could be an empty list). + // The server will continue to read and respond to subsequent ReadRequests until the client closes the stream. + // Once it has sent all the datum, the server will send a ReadResponse with the end of transmission flag set to true. + rpc ReadFn(stream ReadRequest) returns (stream ReadResponse); - // AckFn acknowledges a list of datum offsets. + // AckFn acknowledges a stream of datum offsets. // When AckFn is called, it implicitly indicates that the datum stream has been processed by the source vertex. // The caller (numa) expects the AckFn to be successful, and it does not expect any errors. // If there are some irrecoverable errors when the callee (UDSource) is processing the AckFn request, - // then it is best to crash because there are no other retry mechanisms possible. - rpc AckFn(AckRequest) returns (AckResponse); + // then it is best to crash because there are no other retry mechanisms possible. + // Clients sends n requests and expects n responses. + rpc AckFn(stream AckRequest) returns (stream AckResponse); // PendingFn returns the number of pending records at the user defined source. rpc PendingFn(google.protobuf.Empty) returns (PendingResponse); @@ -30,6 +33,14 @@ service Source { rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); } +/* + * Handshake message between client and server to indicate the start of transmission. + */ +message Handshake { + // Required field indicating the start of transmission. + bool sot = 1; +} + /* * ReadRequest is the request for reading datum stream from user defined source. */ @@ -44,6 +55,7 @@ message ReadRequest { } // Required field indicating the request. Request request = 1; + optional Handshake handshake = 2; } /* @@ -67,8 +79,31 @@ message ReadResponse { // e.g. Kafka and Redis Stream message usually include information about the headers. map headers = 5; } + message Status { + // Code to indicate the status of the response. + enum Code { + SUCCESS = 0; + FAILURE = 1; + } + + // Error to indicate the error type. If the code is FAILURE, then the error field will be populated. + enum Error { + UNACKED = 0; + OTHER = 1; + } + + // End of transmission flag. + bool eot = 1; + Code code = 2; + optional Error error = 3; + optional string msg = 4; + } // Required field holding the result. Result result = 1; + // Status of the response. Holds the end of transmission flag and the status code. + Status status = 2; + // Handshake message between client and server to indicate the start of transmission. + optional Handshake handshake = 3; } /* @@ -77,14 +112,12 @@ message ReadResponse { */ message AckRequest { message Request { - // Required field holding a list of offsets to be acknowledged. - // The offsets must be strictly corresponding to the previously read batch, - // meaning the offsets must be in the same order as the datum responses in the ReadResponse. - // By enforcing ordering, we can save deserialization effort on the server side, assuming the server keeps a local copy of the raw/un-serialized offsets. + // Required field holding the offset to be acked repeated Offset offsets = 1; } // Required field holding the request. The list will be ordered and will have the same order as the original Read response. Request request = 1; + optional Handshake handshake = 2; } /* @@ -104,6 +137,8 @@ message AckResponse { } // Required field holding the result. Result result = 1; + // Handshake message between client and server to indicate the start of transmission. + optional Handshake handshake = 2; } /* diff --git a/pkg/apis/proto/source/v1/source_grpc.pb.go b/pkg/apis/proto/source/v1/source_grpc.pb.go index ebb38e85..69818db4 100644 --- a/pkg/apis/proto/source/v1/source_grpc.pb.go +++ b/pkg/apis/proto/source/v1/source_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v4.25.1 // source: pkg/apis/proto/source/v1/source.proto @@ -16,23 +16,34 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + Source_ReadFn_FullMethodName = "/source.v1.Source/ReadFn" + Source_AckFn_FullMethodName = "/source.v1.Source/AckFn" + Source_PendingFn_FullMethodName = "/source.v1.Source/PendingFn" + Source_PartitionsFn_FullMethodName = "/source.v1.Source/PartitionsFn" + Source_IsReady_FullMethodName = "/source.v1.Source/IsReady" +) // SourceClient is the client API for Source service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type SourceClient interface { // Read returns a stream of datum responses. - // The size of the returned ReadResponse is less than or equal to the num_records specified in ReadRequest. - // If the request timeout is reached on server side, the returned ReadResponse will contain all the datum that have been read (which could be an empty list). - ReadFn(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (Source_ReadFnClient, error) - // AckFn acknowledges a list of datum offsets. + // The size of the returned responses is less than or equal to the num_records specified in each ReadRequest. + // If the request timeout is reached on the server side, the returned responses will contain all the datum that have been read (which could be an empty list). + // The server will continue to read and respond to subsequent ReadRequests until the client closes the stream. + // Once it has sent all the datum, the server will send a ReadResponse with the end of transmission flag set to true. + ReadFn(ctx context.Context, opts ...grpc.CallOption) (Source_ReadFnClient, error) + // AckFn acknowledges a stream of datum offsets. // When AckFn is called, it implicitly indicates that the datum stream has been processed by the source vertex. // The caller (numa) expects the AckFn to be successful, and it does not expect any errors. // If there are some irrecoverable errors when the callee (UDSource) is processing the AckFn request, // then it is best to crash because there are no other retry mechanisms possible. - AckFn(ctx context.Context, in *AckRequest, opts ...grpc.CallOption) (*AckResponse, error) + // Clients sends n requests and expects n responses. + AckFn(ctx context.Context, opts ...grpc.CallOption) (Source_AckFnClient, error) // PendingFn returns the number of pending records at the user defined source. PendingFn(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*PendingResponse, error) // PartitionsFn returns the list of partitions for the user defined source. @@ -49,22 +60,18 @@ func NewSourceClient(cc grpc.ClientConnInterface) SourceClient { return &sourceClient{cc} } -func (c *sourceClient) ReadFn(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (Source_ReadFnClient, error) { - stream, err := c.cc.NewStream(ctx, &Source_ServiceDesc.Streams[0], "/source.v1.Source/ReadFn", opts...) +func (c *sourceClient) ReadFn(ctx context.Context, opts ...grpc.CallOption) (Source_ReadFnClient, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Source_ServiceDesc.Streams[0], Source_ReadFn_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &sourceReadFnClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } + x := &sourceReadFnClient{ClientStream: stream} return x, nil } type Source_ReadFnClient interface { + Send(*ReadRequest) error Recv() (*ReadResponse, error) grpc.ClientStream } @@ -73,6 +80,10 @@ type sourceReadFnClient struct { grpc.ClientStream } +func (x *sourceReadFnClient) Send(m *ReadRequest) error { + return x.ClientStream.SendMsg(m) +} + func (x *sourceReadFnClient) Recv() (*ReadResponse, error) { m := new(ReadResponse) if err := x.ClientStream.RecvMsg(m); err != nil { @@ -81,18 +92,42 @@ func (x *sourceReadFnClient) Recv() (*ReadResponse, error) { return m, nil } -func (c *sourceClient) AckFn(ctx context.Context, in *AckRequest, opts ...grpc.CallOption) (*AckResponse, error) { - out := new(AckResponse) - err := c.cc.Invoke(ctx, "/source.v1.Source/AckFn", in, out, opts...) +func (c *sourceClient) AckFn(ctx context.Context, opts ...grpc.CallOption) (Source_AckFnClient, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Source_ServiceDesc.Streams[1], Source_AckFn_FullMethodName, cOpts...) if err != nil { return nil, err } - return out, nil + x := &sourceAckFnClient{ClientStream: stream} + return x, nil +} + +type Source_AckFnClient interface { + Send(*AckRequest) error + Recv() (*AckResponse, error) + grpc.ClientStream +} + +type sourceAckFnClient struct { + grpc.ClientStream +} + +func (x *sourceAckFnClient) Send(m *AckRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *sourceAckFnClient) Recv() (*AckResponse, error) { + m := new(AckResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil } func (c *sourceClient) PendingFn(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*PendingResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PendingResponse) - err := c.cc.Invoke(ctx, "/source.v1.Source/PendingFn", in, out, opts...) + err := c.cc.Invoke(ctx, Source_PendingFn_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -100,8 +135,9 @@ func (c *sourceClient) PendingFn(ctx context.Context, in *emptypb.Empty, opts .. } func (c *sourceClient) PartitionsFn(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*PartitionsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PartitionsResponse) - err := c.cc.Invoke(ctx, "/source.v1.Source/PartitionsFn", in, out, opts...) + err := c.cc.Invoke(ctx, Source_PartitionsFn_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -109,8 +145,9 @@ func (c *sourceClient) PartitionsFn(ctx context.Context, in *emptypb.Empty, opts } func (c *sourceClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ReadyResponse) - err := c.cc.Invoke(ctx, "/source.v1.Source/IsReady", in, out, opts...) + err := c.cc.Invoke(ctx, Source_IsReady_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -122,15 +159,18 @@ func (c *sourceClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...g // for forward compatibility type SourceServer interface { // Read returns a stream of datum responses. - // The size of the returned ReadResponse is less than or equal to the num_records specified in ReadRequest. - // If the request timeout is reached on server side, the returned ReadResponse will contain all the datum that have been read (which could be an empty list). - ReadFn(*ReadRequest, Source_ReadFnServer) error - // AckFn acknowledges a list of datum offsets. + // The size of the returned responses is less than or equal to the num_records specified in each ReadRequest. + // If the request timeout is reached on the server side, the returned responses will contain all the datum that have been read (which could be an empty list). + // The server will continue to read and respond to subsequent ReadRequests until the client closes the stream. + // Once it has sent all the datum, the server will send a ReadResponse with the end of transmission flag set to true. + ReadFn(Source_ReadFnServer) error + // AckFn acknowledges a stream of datum offsets. // When AckFn is called, it implicitly indicates that the datum stream has been processed by the source vertex. // The caller (numa) expects the AckFn to be successful, and it does not expect any errors. // If there are some irrecoverable errors when the callee (UDSource) is processing the AckFn request, // then it is best to crash because there are no other retry mechanisms possible. - AckFn(context.Context, *AckRequest) (*AckResponse, error) + // Clients sends n requests and expects n responses. + AckFn(Source_AckFnServer) error // PendingFn returns the number of pending records at the user defined source. PendingFn(context.Context, *emptypb.Empty) (*PendingResponse, error) // PartitionsFn returns the list of partitions for the user defined source. @@ -144,11 +184,11 @@ type SourceServer interface { type UnimplementedSourceServer struct { } -func (UnimplementedSourceServer) ReadFn(*ReadRequest, Source_ReadFnServer) error { +func (UnimplementedSourceServer) ReadFn(Source_ReadFnServer) error { return status.Errorf(codes.Unimplemented, "method ReadFn not implemented") } -func (UnimplementedSourceServer) AckFn(context.Context, *AckRequest) (*AckResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method AckFn not implemented") +func (UnimplementedSourceServer) AckFn(Source_AckFnServer) error { + return status.Errorf(codes.Unimplemented, "method AckFn not implemented") } func (UnimplementedSourceServer) PendingFn(context.Context, *emptypb.Empty) (*PendingResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method PendingFn not implemented") @@ -173,15 +213,12 @@ func RegisterSourceServer(s grpc.ServiceRegistrar, srv SourceServer) { } func _Source_ReadFn_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(ReadRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(SourceServer).ReadFn(m, &sourceReadFnServer{stream}) + return srv.(SourceServer).ReadFn(&sourceReadFnServer{ServerStream: stream}) } type Source_ReadFnServer interface { Send(*ReadResponse) error + Recv() (*ReadRequest, error) grpc.ServerStream } @@ -193,22 +230,38 @@ func (x *sourceReadFnServer) Send(m *ReadResponse) error { return x.ServerStream.SendMsg(m) } -func _Source_AckFn_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(AckRequest) - if err := dec(in); err != nil { +func (x *sourceReadFnServer) Recv() (*ReadRequest, error) { + m := new(ReadRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { return nil, err } - if interceptor == nil { - return srv.(SourceServer).AckFn(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/source.v1.Source/AckFn", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SourceServer).AckFn(ctx, req.(*AckRequest)) + return m, nil +} + +func _Source_AckFn_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SourceServer).AckFn(&sourceAckFnServer{ServerStream: stream}) +} + +type Source_AckFnServer interface { + Send(*AckResponse) error + Recv() (*AckRequest, error) + grpc.ServerStream +} + +type sourceAckFnServer struct { + grpc.ServerStream +} + +func (x *sourceAckFnServer) Send(m *AckResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *sourceAckFnServer) Recv() (*AckRequest, error) { + m := new(AckRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err } - return interceptor(ctx, in, info, handler) + return m, nil } func _Source_PendingFn_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { @@ -221,7 +274,7 @@ func _Source_PendingFn_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/source.v1.Source/PendingFn", + FullMethod: Source_PendingFn_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(SourceServer).PendingFn(ctx, req.(*emptypb.Empty)) @@ -239,7 +292,7 @@ func _Source_PartitionsFn_Handler(srv interface{}, ctx context.Context, dec func } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/source.v1.Source/PartitionsFn", + FullMethod: Source_PartitionsFn_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(SourceServer).PartitionsFn(ctx, req.(*emptypb.Empty)) @@ -257,7 +310,7 @@ func _Source_IsReady_Handler(srv interface{}, ctx context.Context, dec func(inte } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/source.v1.Source/IsReady", + FullMethod: Source_IsReady_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(SourceServer).IsReady(ctx, req.(*emptypb.Empty)) @@ -272,10 +325,6 @@ var Source_ServiceDesc = grpc.ServiceDesc{ ServiceName: "source.v1.Source", HandlerType: (*SourceServer)(nil), Methods: []grpc.MethodDesc{ - { - MethodName: "AckFn", - Handler: _Source_AckFn_Handler, - }, { MethodName: "PendingFn", Handler: _Source_PendingFn_Handler, @@ -294,6 +343,13 @@ var Source_ServiceDesc = grpc.ServiceDesc{ StreamName: "ReadFn", Handler: _Source_ReadFn_Handler, ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "AckFn", + Handler: _Source_AckFn_Handler, + ServerStreams: true, + ClientStreams: true, }, }, Metadata: "pkg/apis/proto/source/v1/source.proto", diff --git a/pkg/apis/proto/source/v1/sourcemock/sourcemock.go b/pkg/apis/proto/source/v1/sourcemock/sourcemock.go index 6d23e3d1..b868dc82 100644 --- a/pkg/apis/proto/source/v1/sourcemock/sourcemock.go +++ b/pkg/apis/proto/source/v1/sourcemock/sourcemock.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/numaproj/numaflow-go/pkg/apis/proto/source/v1 (interfaces: SourceClient,Source_ReadFnClient) +// Source: github.com/numaproj/numaflow-go/pkg/apis/proto/source/v1 (interfaces: SourceClient,Source_ReadFnClient,Source_AckFnClient) // Package sourcemock is a generated GoMock package. package sourcemock @@ -39,22 +39,22 @@ func (m *MockSourceClient) EXPECT() *MockSourceClientMockRecorder { } // AckFn mocks base method. -func (m *MockSourceClient) AckFn(arg0 context.Context, arg1 *v1.AckRequest, arg2 ...grpc.CallOption) (*v1.AckResponse, error) { +func (m *MockSourceClient) AckFn(arg0 context.Context, arg1 ...grpc.CallOption) (v1.Source_AckFnClient, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []interface{}{arg0} + for _, a := range arg1 { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "AckFn", varargs...) - ret0, _ := ret[0].(*v1.AckResponse) + ret0, _ := ret[0].(v1.Source_AckFnClient) ret1, _ := ret[1].(error) return ret0, ret1 } // AckFn indicates an expected call of AckFn. -func (mr *MockSourceClientMockRecorder) AckFn(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSourceClientMockRecorder) AckFn(arg0 interface{}, arg1 ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]interface{}{arg0}, arg1...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AckFn", reflect.TypeOf((*MockSourceClient)(nil).AckFn), varargs...) } @@ -119,10 +119,10 @@ func (mr *MockSourceClientMockRecorder) PendingFn(arg0, arg1 interface{}, arg2 . } // ReadFn mocks base method. -func (m *MockSourceClient) ReadFn(arg0 context.Context, arg1 *v1.ReadRequest, arg2 ...grpc.CallOption) (v1.Source_ReadFnClient, error) { +func (m *MockSourceClient) ReadFn(arg0 context.Context, arg1 ...grpc.CallOption) (v1.Source_ReadFnClient, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []interface{}{arg0} + for _, a := range arg1 { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "ReadFn", varargs...) @@ -132,9 +132,9 @@ func (m *MockSourceClient) ReadFn(arg0 context.Context, arg1 *v1.ReadRequest, ar } // ReadFn indicates an expected call of ReadFn. -func (mr *MockSourceClientMockRecorder) ReadFn(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSourceClientMockRecorder) ReadFn(arg0 interface{}, arg1 ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]interface{}{arg0}, arg1...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadFn", reflect.TypeOf((*MockSourceClient)(nil).ReadFn), varargs...) } @@ -233,6 +233,20 @@ func (mr *MockSource_ReadFnClientMockRecorder) RecvMsg(arg0 interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockSource_ReadFnClient)(nil).RecvMsg), arg0) } +// Send mocks base method. +func (m *MockSource_ReadFnClient) Send(arg0 *v1.ReadRequest) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send. +func (mr *MockSource_ReadFnClientMockRecorder) Send(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockSource_ReadFnClient)(nil).Send), arg0) +} + // SendMsg mocks base method. func (m *MockSource_ReadFnClient) SendMsg(arg0 interface{}) error { m.ctrl.T.Helper() @@ -260,3 +274,140 @@ func (mr *MockSource_ReadFnClientMockRecorder) Trailer() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Trailer", reflect.TypeOf((*MockSource_ReadFnClient)(nil).Trailer)) } + +// MockSource_AckFnClient is a mock of Source_AckFnClient interface. +type MockSource_AckFnClient struct { + ctrl *gomock.Controller + recorder *MockSource_AckFnClientMockRecorder +} + +// MockSource_AckFnClientMockRecorder is the mock recorder for MockSource_AckFnClient. +type MockSource_AckFnClientMockRecorder struct { + mock *MockSource_AckFnClient +} + +// NewMockSource_AckFnClient creates a new mock instance. +func NewMockSource_AckFnClient(ctrl *gomock.Controller) *MockSource_AckFnClient { + mock := &MockSource_AckFnClient{ctrl: ctrl} + mock.recorder = &MockSource_AckFnClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSource_AckFnClient) EXPECT() *MockSource_AckFnClientMockRecorder { + return m.recorder +} + +// CloseSend mocks base method. +func (m *MockSource_AckFnClient) CloseSend() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CloseSend") + ret0, _ := ret[0].(error) + return ret0 +} + +// CloseSend indicates an expected call of CloseSend. +func (mr *MockSource_AckFnClientMockRecorder) CloseSend() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloseSend", reflect.TypeOf((*MockSource_AckFnClient)(nil).CloseSend)) +} + +// Context mocks base method. +func (m *MockSource_AckFnClient) Context() context.Context { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Context") + ret0, _ := ret[0].(context.Context) + return ret0 +} + +// Context indicates an expected call of Context. +func (mr *MockSource_AckFnClientMockRecorder) Context() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Context", reflect.TypeOf((*MockSource_AckFnClient)(nil).Context)) +} + +// Header mocks base method. +func (m *MockSource_AckFnClient) Header() (metadata.MD, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Header") + ret0, _ := ret[0].(metadata.MD) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Header indicates an expected call of Header. +func (mr *MockSource_AckFnClientMockRecorder) Header() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Header", reflect.TypeOf((*MockSource_AckFnClient)(nil).Header)) +} + +// Recv mocks base method. +func (m *MockSource_AckFnClient) Recv() (*v1.AckResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Recv") + ret0, _ := ret[0].(*v1.AckResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Recv indicates an expected call of Recv. +func (mr *MockSource_AckFnClientMockRecorder) Recv() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Recv", reflect.TypeOf((*MockSource_AckFnClient)(nil).Recv)) +} + +// RecvMsg mocks base method. +func (m *MockSource_AckFnClient) RecvMsg(arg0 interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RecvMsg", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// RecvMsg indicates an expected call of RecvMsg. +func (mr *MockSource_AckFnClientMockRecorder) RecvMsg(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RecvMsg", reflect.TypeOf((*MockSource_AckFnClient)(nil).RecvMsg), arg0) +} + +// Send mocks base method. +func (m *MockSource_AckFnClient) Send(arg0 *v1.AckRequest) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send. +func (mr *MockSource_AckFnClientMockRecorder) Send(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockSource_AckFnClient)(nil).Send), arg0) +} + +// SendMsg mocks base method. +func (m *MockSource_AckFnClient) SendMsg(arg0 interface{}) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SendMsg", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// SendMsg indicates an expected call of SendMsg. +func (mr *MockSource_AckFnClientMockRecorder) SendMsg(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendMsg", reflect.TypeOf((*MockSource_AckFnClient)(nil).SendMsg), arg0) +} + +// Trailer mocks base method. +func (m *MockSource_AckFnClient) Trailer() metadata.MD { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Trailer") + ret0, _ := ret[0].(metadata.MD) + return ret0 +} + +// Trailer indicates an expected call of Trailer. +func (mr *MockSource_AckFnClientMockRecorder) Trailer() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Trailer", reflect.TypeOf((*MockSource_AckFnClient)(nil).Trailer)) +} diff --git a/pkg/apis/proto/sourcetransform/v1/transform.pb.go b/pkg/apis/proto/sourcetransform/v1/transform.pb.go index b56efc20..e851979e 100644 --- a/pkg/apis/proto/sourcetransform/v1/transform.pb.go +++ b/pkg/apis/proto/sourcetransform/v1/transform.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.33.0 +// protoc-gen-go v1.34.2 // protoc v4.25.1 // source: pkg/apis/proto/sourcetransform/v1/transform.proto @@ -22,22 +22,18 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) -// * -// SourceTransformerRequest represents a request element. -type SourceTransformRequest struct { +// Handshake message between client and server to indicate the start of transmission. +type Handshake struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` - Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` - Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` - Headers map[string]string `protobuf:"bytes,5,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // Required field indicating the start of transmission. + Sot bool `protobuf:"varint,1,opt,name=sot,proto3" json:"sot,omitempty"` } -func (x *SourceTransformRequest) Reset() { - *x = SourceTransformRequest{} +func (x *Handshake) Reset() { + *x = Handshake{} if protoimpl.UnsafeEnabled { mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[0] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -45,13 +41,13 @@ func (x *SourceTransformRequest) Reset() { } } -func (x *SourceTransformRequest) String() string { +func (x *Handshake) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SourceTransformRequest) ProtoMessage() {} +func (*Handshake) ProtoMessage() {} -func (x *SourceTransformRequest) ProtoReflect() protoreflect.Message { +func (x *Handshake) ProtoReflect() protoreflect.Message { mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[0] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -63,42 +59,71 @@ func (x *SourceTransformRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use SourceTransformRequest.ProtoReflect.Descriptor instead. -func (*SourceTransformRequest) Descriptor() ([]byte, []int) { +// Deprecated: Use Handshake.ProtoReflect.Descriptor instead. +func (*Handshake) Descriptor() ([]byte, []int) { return file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDescGZIP(), []int{0} } -func (x *SourceTransformRequest) GetKeys() []string { +func (x *Handshake) GetSot() bool { if x != nil { - return x.Keys + return x.Sot } - return nil + return false } -func (x *SourceTransformRequest) GetValue() []byte { - if x != nil { - return x.Value +// * +// SourceTransformerRequest represents a request element. +type SourceTransformRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Request *SourceTransformRequest_Request `protobuf:"bytes,1,opt,name=request,proto3" json:"request,omitempty"` + Handshake *Handshake `protobuf:"bytes,2,opt,name=handshake,proto3,oneof" json:"handshake,omitempty"` +} + +func (x *SourceTransformRequest) Reset() { + *x = SourceTransformRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } - return nil } -func (x *SourceTransformRequest) GetEventTime() *timestamppb.Timestamp { - if x != nil { - return x.EventTime +func (x *SourceTransformRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceTransformRequest) ProtoMessage() {} + +func (x *SourceTransformRequest) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return nil + return mi.MessageOf(x) +} + +// Deprecated: Use SourceTransformRequest.ProtoReflect.Descriptor instead. +func (*SourceTransformRequest) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDescGZIP(), []int{1} } -func (x *SourceTransformRequest) GetWatermark() *timestamppb.Timestamp { +func (x *SourceTransformRequest) GetRequest() *SourceTransformRequest_Request { if x != nil { - return x.Watermark + return x.Request } return nil } -func (x *SourceTransformRequest) GetHeaders() map[string]string { +func (x *SourceTransformRequest) GetHandshake() *Handshake { if x != nil { - return x.Headers + return x.Handshake } return nil } @@ -111,12 +136,16 @@ type SourceTransformResponse struct { unknownFields protoimpl.UnknownFields Results []*SourceTransformResponse_Result `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + // This ID is used to refer the responses to the request it corresponds to. + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + // Handshake message between client and server to indicate the start of transmission. + Handshake *Handshake `protobuf:"bytes,3,opt,name=handshake,proto3,oneof" json:"handshake,omitempty"` } func (x *SourceTransformResponse) Reset() { *x = SourceTransformResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[1] + mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -129,7 +158,7 @@ func (x *SourceTransformResponse) String() string { func (*SourceTransformResponse) ProtoMessage() {} func (x *SourceTransformResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[1] + mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[2] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -142,7 +171,7 @@ func (x *SourceTransformResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SourceTransformResponse.ProtoReflect.Descriptor instead. func (*SourceTransformResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDescGZIP(), []int{1} + return file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDescGZIP(), []int{2} } func (x *SourceTransformResponse) GetResults() []*SourceTransformResponse_Result { @@ -152,6 +181,20 @@ func (x *SourceTransformResponse) GetResults() []*SourceTransformResponse_Result return nil } +func (x *SourceTransformResponse) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *SourceTransformResponse) GetHandshake() *Handshake { + if x != nil { + return x.Handshake + } + return nil +} + // * // ReadyResponse is the health check result. type ReadyResponse struct { @@ -165,7 +208,7 @@ type ReadyResponse struct { func (x *ReadyResponse) Reset() { *x = ReadyResponse{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[2] + mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -178,7 +221,7 @@ func (x *ReadyResponse) String() string { func (*ReadyResponse) ProtoMessage() {} func (x *ReadyResponse) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[2] + mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[3] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -191,7 +234,7 @@ func (x *ReadyResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReadyResponse.ProtoReflect.Descriptor instead. func (*ReadyResponse) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDescGZIP(), []int{2} + return file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDescGZIP(), []int{3} } func (x *ReadyResponse) GetReady() bool { @@ -201,6 +244,94 @@ func (x *ReadyResponse) GetReady() bool { return false } +type SourceTransformRequest_Request struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Keys []string `protobuf:"bytes,1,rep,name=keys,proto3" json:"keys,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + EventTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=event_time,json=eventTime,proto3" json:"event_time,omitempty"` + Watermark *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=watermark,proto3" json:"watermark,omitempty"` + Headers map[string]string `protobuf:"bytes,5,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + // This ID is used to uniquely identify a transform request + Id string `protobuf:"bytes,6,opt,name=id,proto3" json:"id,omitempty"` +} + +func (x *SourceTransformRequest_Request) Reset() { + *x = SourceTransformRequest_Request{} + if protoimpl.UnsafeEnabled { + mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SourceTransformRequest_Request) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceTransformRequest_Request) ProtoMessage() {} + +func (x *SourceTransformRequest_Request) ProtoReflect() protoreflect.Message { + mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SourceTransformRequest_Request.ProtoReflect.Descriptor instead. +func (*SourceTransformRequest_Request) Descriptor() ([]byte, []int) { + return file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDescGZIP(), []int{1, 0} +} + +func (x *SourceTransformRequest_Request) GetKeys() []string { + if x != nil { + return x.Keys + } + return nil +} + +func (x *SourceTransformRequest_Request) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +func (x *SourceTransformRequest_Request) GetEventTime() *timestamppb.Timestamp { + if x != nil { + return x.EventTime + } + return nil +} + +func (x *SourceTransformRequest_Request) GetWatermark() *timestamppb.Timestamp { + if x != nil { + return x.Watermark + } + return nil +} + +func (x *SourceTransformRequest_Request) GetHeaders() map[string]string { + if x != nil { + return x.Headers + } + return nil +} + +func (x *SourceTransformRequest_Request) GetId() string { + if x != nil { + return x.Id + } + return "" +} + type SourceTransformResponse_Result struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -215,7 +346,7 @@ type SourceTransformResponse_Result struct { func (x *SourceTransformResponse_Result) Reset() { *x = SourceTransformResponse_Result{} if protoimpl.UnsafeEnabled { - mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[4] + mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -228,7 +359,7 @@ func (x *SourceTransformResponse_Result) String() string { func (*SourceTransformResponse_Result) ProtoMessage() {} func (x *SourceTransformResponse_Result) ProtoReflect() protoreflect.Message { - mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[4] + mi := &file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -241,7 +372,7 @@ func (x *SourceTransformResponse_Result) ProtoReflect() protoreflect.Message { // Deprecated: Use SourceTransformResponse_Result.ProtoReflect.Descriptor instead. func (*SourceTransformResponse_Result) Descriptor() ([]byte, []int) { - return file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDescGZIP(), []int{1, 0} + return file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDescGZIP(), []int{2, 0} } func (x *SourceTransformResponse_Result) GetKeys() []string { @@ -283,63 +414,83 @@ var file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDesc = []byte{ 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, 0x6d, 0x70, 0x74, - 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc8, 0x02, 0x0a, 0x16, 0x53, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, + 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x1d, 0x0a, 0x09, 0x48, 0x61, 0x6e, 0x64, 0x73, + 0x68, 0x61, 0x6b, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x03, 0x73, 0x6f, 0x74, 0x22, 0x8e, 0x04, 0x0a, 0x16, 0x53, 0x6f, 0x75, 0x72, 0x63, + 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x4e, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x42, 0x0a, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, 0x6e, 0x64, + 0x73, 0x68, 0x61, 0x6b, 0x65, 0x48, 0x00, 0x52, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, + 0x6b, 0x65, 0x88, 0x01, 0x01, 0x1a, 0xd1, 0x02, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, + 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, + 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, + 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, 0x6b, + 0x12, 0x5b, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x41, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, + 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x0e, 0x0a, + 0x02, 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x1a, 0x3a, 0x0a, + 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x68, 0x61, + 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x22, 0xcf, 0x02, 0x0a, 0x17, 0x53, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x34, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, 0x72, 0x61, + 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x73, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x02, 0x69, 0x64, 0x12, 0x42, 0x0a, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, + 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x48, 0x61, + 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x48, 0x00, 0x52, 0x09, 0x68, 0x61, 0x6e, 0x64, 0x73, + 0x68, 0x61, 0x6b, 0x65, 0x88, 0x01, 0x01, 0x1a, 0x81, 0x01, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, + 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, - 0x6d, 0x61, 0x72, 0x6b, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x77, 0x61, 0x74, 0x65, 0x72, 0x6d, 0x61, 0x72, - 0x6b, 0x12, 0x53, 0x0a, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x39, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, - 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, - 0x38, 0x01, 0x22, 0xed, 0x01, 0x0a, 0x17, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, - 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x34, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, - 0x6d, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, - 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x1a, 0x81, - 0x01, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6b, 0x65, 0x79, - 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x6b, 0x65, 0x79, 0x73, 0x12, 0x14, 0x0a, - 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, - 0x6c, 0x75, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, - 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x12, - 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, - 0x67, 0x73, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x32, 0xcb, 0x01, 0x0a, 0x0f, 0x53, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x70, 0x0a, - 0x11, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, - 0x46, 0x6e, 0x12, 0x2c, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x2d, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, - 0x72, 0x6d, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, - 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x46, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x1a, 0x23, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, - 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x43, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, - 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, - 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, - 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x6e, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, + 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, + 0x68, 0x61, 0x6e, 0x64, 0x73, 0x68, 0x61, 0x6b, 0x65, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, + 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, + 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, + 0x32, 0xcf, 0x01, 0x0a, 0x0f, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x74, 0x0a, 0x11, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, + 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x46, 0x6e, 0x12, 0x2c, 0x2e, 0x73, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x2e, 0x76, 0x31, + 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2d, 0x2e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x53, + 0x6f, 0x75, 0x72, 0x63, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, 0x30, 0x01, 0x12, 0x46, 0x0a, 0x07, 0x49, 0x73, + 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x23, 0x2e, + 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x66, 0x6f, 0x72, 0x6d, 0x65, + 0x72, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, + 0x73, 0x65, 0x42, 0x43, 0x5a, 0x41, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, + 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x74, 0x72, 0x61, 0x6e, 0x73, + 0x66, 0x6f, 0x72, 0x6d, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -354,31 +505,36 @@ func file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDescGZIP() []byte return file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDescData } -var file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes = make([]protoimpl.MessageInfo, 5) -var file_pkg_apis_proto_sourcetransform_v1_transform_proto_goTypes = []interface{}{ - (*SourceTransformRequest)(nil), // 0: sourcetransformer.v1.SourceTransformRequest - (*SourceTransformResponse)(nil), // 1: sourcetransformer.v1.SourceTransformResponse - (*ReadyResponse)(nil), // 2: sourcetransformer.v1.ReadyResponse - nil, // 3: sourcetransformer.v1.SourceTransformRequest.HeadersEntry - (*SourceTransformResponse_Result)(nil), // 4: sourcetransformer.v1.SourceTransformResponse.Result - (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 6: google.protobuf.Empty +var file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_pkg_apis_proto_sourcetransform_v1_transform_proto_goTypes = []any{ + (*Handshake)(nil), // 0: sourcetransformer.v1.Handshake + (*SourceTransformRequest)(nil), // 1: sourcetransformer.v1.SourceTransformRequest + (*SourceTransformResponse)(nil), // 2: sourcetransformer.v1.SourceTransformResponse + (*ReadyResponse)(nil), // 3: sourcetransformer.v1.ReadyResponse + (*SourceTransformRequest_Request)(nil), // 4: sourcetransformer.v1.SourceTransformRequest.Request + nil, // 5: sourcetransformer.v1.SourceTransformRequest.Request.HeadersEntry + (*SourceTransformResponse_Result)(nil), // 6: sourcetransformer.v1.SourceTransformResponse.Result + (*timestamppb.Timestamp)(nil), // 7: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 8: google.protobuf.Empty } var file_pkg_apis_proto_sourcetransform_v1_transform_proto_depIdxs = []int32{ - 5, // 0: sourcetransformer.v1.SourceTransformRequest.event_time:type_name -> google.protobuf.Timestamp - 5, // 1: sourcetransformer.v1.SourceTransformRequest.watermark:type_name -> google.protobuf.Timestamp - 3, // 2: sourcetransformer.v1.SourceTransformRequest.headers:type_name -> sourcetransformer.v1.SourceTransformRequest.HeadersEntry - 4, // 3: sourcetransformer.v1.SourceTransformResponse.results:type_name -> sourcetransformer.v1.SourceTransformResponse.Result - 5, // 4: sourcetransformer.v1.SourceTransformResponse.Result.event_time:type_name -> google.protobuf.Timestamp - 0, // 5: sourcetransformer.v1.SourceTransform.SourceTransformFn:input_type -> sourcetransformer.v1.SourceTransformRequest - 6, // 6: sourcetransformer.v1.SourceTransform.IsReady:input_type -> google.protobuf.Empty - 1, // 7: sourcetransformer.v1.SourceTransform.SourceTransformFn:output_type -> sourcetransformer.v1.SourceTransformResponse - 2, // 8: sourcetransformer.v1.SourceTransform.IsReady:output_type -> sourcetransformer.v1.ReadyResponse - 7, // [7:9] is the sub-list for method output_type - 5, // [5:7] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 4, // 0: sourcetransformer.v1.SourceTransformRequest.request:type_name -> sourcetransformer.v1.SourceTransformRequest.Request + 0, // 1: sourcetransformer.v1.SourceTransformRequest.handshake:type_name -> sourcetransformer.v1.Handshake + 6, // 2: sourcetransformer.v1.SourceTransformResponse.results:type_name -> sourcetransformer.v1.SourceTransformResponse.Result + 0, // 3: sourcetransformer.v1.SourceTransformResponse.handshake:type_name -> sourcetransformer.v1.Handshake + 7, // 4: sourcetransformer.v1.SourceTransformRequest.Request.event_time:type_name -> google.protobuf.Timestamp + 7, // 5: sourcetransformer.v1.SourceTransformRequest.Request.watermark:type_name -> google.protobuf.Timestamp + 5, // 6: sourcetransformer.v1.SourceTransformRequest.Request.headers:type_name -> sourcetransformer.v1.SourceTransformRequest.Request.HeadersEntry + 7, // 7: sourcetransformer.v1.SourceTransformResponse.Result.event_time:type_name -> google.protobuf.Timestamp + 1, // 8: sourcetransformer.v1.SourceTransform.SourceTransformFn:input_type -> sourcetransformer.v1.SourceTransformRequest + 8, // 9: sourcetransformer.v1.SourceTransform.IsReady:input_type -> google.protobuf.Empty + 2, // 10: sourcetransformer.v1.SourceTransform.SourceTransformFn:output_type -> sourcetransformer.v1.SourceTransformResponse + 3, // 11: sourcetransformer.v1.SourceTransform.IsReady:output_type -> sourcetransformer.v1.ReadyResponse + 10, // [10:12] is the sub-list for method output_type + 8, // [8:10] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name } func init() { file_pkg_apis_proto_sourcetransform_v1_transform_proto_init() } @@ -387,7 +543,19 @@ func file_pkg_apis_proto_sourcetransform_v1_transform_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[0].Exporter = func(v any, i int) any { + switch v := v.(*Handshake); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*SourceTransformRequest); i { case 0: return &v.state @@ -399,7 +567,7 @@ func file_pkg_apis_proto_sourcetransform_v1_transform_proto_init() { return nil } } - file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*SourceTransformResponse); i { case 0: return &v.state @@ -411,7 +579,7 @@ func file_pkg_apis_proto_sourcetransform_v1_transform_proto_init() { return nil } } - file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*ReadyResponse); i { case 0: return &v.state @@ -423,7 +591,19 @@ func file_pkg_apis_proto_sourcetransform_v1_transform_proto_init() { return nil } } - file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[4].Exporter = func(v any, i int) any { + switch v := v.(*SourceTransformRequest_Request); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[6].Exporter = func(v any, i int) any { switch v := v.(*SourceTransformResponse_Result); i { case 0: return &v.state @@ -436,13 +616,15 @@ func file_pkg_apis_proto_sourcetransform_v1_transform_proto_init() { } } } + file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[1].OneofWrappers = []any{} + file_pkg_apis_proto_sourcetransform_v1_transform_proto_msgTypes[2].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_apis_proto_sourcetransform_v1_transform_proto_rawDesc, NumEnums: 0, - NumMessages: 5, + NumMessages: 7, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/apis/proto/sourcetransform/v1/transform.proto b/pkg/apis/proto/sourcetransform/v1/transform.proto index 8885db08..7eec50c9 100644 --- a/pkg/apis/proto/sourcetransform/v1/transform.proto +++ b/pkg/apis/proto/sourcetransform/v1/transform.proto @@ -11,21 +11,35 @@ service SourceTransform { // SourceTransformFn applies a function to each request element. // In addition to map function, SourceTransformFn also supports assigning a new event time to response. // SourceTransformFn can be used only at source vertex by source data transformer. - rpc SourceTransformFn(SourceTransformRequest) returns (SourceTransformResponse); + rpc SourceTransformFn(stream SourceTransformRequest) returns (stream SourceTransformResponse); // IsReady is the heartbeat endpoint for gRPC. rpc IsReady(google.protobuf.Empty) returns (ReadyResponse); } +/* + * Handshake message between client and server to indicate the start of transmission. + */ + message Handshake { + // Required field indicating the start of transmission. + bool sot = 1; +} + /** * SourceTransformerRequest represents a request element. */ message SourceTransformRequest { - repeated string keys = 1; - bytes value = 2; - google.protobuf.Timestamp event_time = 3; - google.protobuf.Timestamp watermark = 4; - map headers = 5; + message Request { + repeated string keys = 1; + bytes value = 2; + google.protobuf.Timestamp event_time = 3; + google.protobuf.Timestamp watermark = 4; + map headers = 5; + // This ID is used to uniquely identify a transform request + string id = 6; + } + Request request = 1; + optional Handshake handshake = 2; } /** @@ -39,6 +53,10 @@ message SourceTransformResponse { repeated string tags = 4; } repeated Result results = 1; + // This ID is used to refer the responses to the request it corresponds to. + string id = 2; + // Handshake message between client and server to indicate the start of transmission. + optional Handshake handshake = 3; } /** diff --git a/pkg/apis/proto/sourcetransform/v1/transform_grpc.pb.go b/pkg/apis/proto/sourcetransform/v1/transform_grpc.pb.go index a3f4c206..52995c34 100644 --- a/pkg/apis/proto/sourcetransform/v1/transform_grpc.pb.go +++ b/pkg/apis/proto/sourcetransform/v1/transform_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.2.0 +// - protoc-gen-go-grpc v1.4.0 // - protoc v4.25.1 // source: pkg/apis/proto/sourcetransform/v1/transform.proto @@ -16,8 +16,13 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + SourceTransform_SourceTransformFn_FullMethodName = "/sourcetransformer.v1.SourceTransform/SourceTransformFn" + SourceTransform_IsReady_FullMethodName = "/sourcetransformer.v1.SourceTransform/IsReady" +) // SourceTransformClient is the client API for SourceTransform service. // @@ -26,7 +31,7 @@ type SourceTransformClient interface { // SourceTransformFn applies a function to each request element. // In addition to map function, SourceTransformFn also supports assigning a new event time to response. // SourceTransformFn can be used only at source vertex by source data transformer. - SourceTransformFn(ctx context.Context, in *SourceTransformRequest, opts ...grpc.CallOption) (*SourceTransformResponse, error) + SourceTransformFn(ctx context.Context, opts ...grpc.CallOption) (SourceTransform_SourceTransformFnClient, error) // IsReady is the heartbeat endpoint for gRPC. IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) } @@ -39,18 +44,42 @@ func NewSourceTransformClient(cc grpc.ClientConnInterface) SourceTransformClient return &sourceTransformClient{cc} } -func (c *sourceTransformClient) SourceTransformFn(ctx context.Context, in *SourceTransformRequest, opts ...grpc.CallOption) (*SourceTransformResponse, error) { - out := new(SourceTransformResponse) - err := c.cc.Invoke(ctx, "/sourcetransformer.v1.SourceTransform/SourceTransformFn", in, out, opts...) +func (c *sourceTransformClient) SourceTransformFn(ctx context.Context, opts ...grpc.CallOption) (SourceTransform_SourceTransformFnClient, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &SourceTransform_ServiceDesc.Streams[0], SourceTransform_SourceTransformFn_FullMethodName, cOpts...) if err != nil { return nil, err } - return out, nil + x := &sourceTransformSourceTransformFnClient{ClientStream: stream} + return x, nil +} + +type SourceTransform_SourceTransformFnClient interface { + Send(*SourceTransformRequest) error + Recv() (*SourceTransformResponse, error) + grpc.ClientStream +} + +type sourceTransformSourceTransformFnClient struct { + grpc.ClientStream +} + +func (x *sourceTransformSourceTransformFnClient) Send(m *SourceTransformRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *sourceTransformSourceTransformFnClient) Recv() (*SourceTransformResponse, error) { + m := new(SourceTransformResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil } func (c *sourceTransformClient) IsReady(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*ReadyResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ReadyResponse) - err := c.cc.Invoke(ctx, "/sourcetransformer.v1.SourceTransform/IsReady", in, out, opts...) + err := c.cc.Invoke(ctx, SourceTransform_IsReady_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -64,7 +93,7 @@ type SourceTransformServer interface { // SourceTransformFn applies a function to each request element. // In addition to map function, SourceTransformFn also supports assigning a new event time to response. // SourceTransformFn can be used only at source vertex by source data transformer. - SourceTransformFn(context.Context, *SourceTransformRequest) (*SourceTransformResponse, error) + SourceTransformFn(SourceTransform_SourceTransformFnServer) error // IsReady is the heartbeat endpoint for gRPC. IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) mustEmbedUnimplementedSourceTransformServer() @@ -74,8 +103,8 @@ type SourceTransformServer interface { type UnimplementedSourceTransformServer struct { } -func (UnimplementedSourceTransformServer) SourceTransformFn(context.Context, *SourceTransformRequest) (*SourceTransformResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method SourceTransformFn not implemented") +func (UnimplementedSourceTransformServer) SourceTransformFn(SourceTransform_SourceTransformFnServer) error { + return status.Errorf(codes.Unimplemented, "method SourceTransformFn not implemented") } func (UnimplementedSourceTransformServer) IsReady(context.Context, *emptypb.Empty) (*ReadyResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method IsReady not implemented") @@ -93,22 +122,30 @@ func RegisterSourceTransformServer(s grpc.ServiceRegistrar, srv SourceTransformS s.RegisterService(&SourceTransform_ServiceDesc, srv) } -func _SourceTransform_SourceTransformFn_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(SourceTransformRequest) - if err := dec(in); err != nil { +func _SourceTransform_SourceTransformFn_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SourceTransformServer).SourceTransformFn(&sourceTransformSourceTransformFnServer{ServerStream: stream}) +} + +type SourceTransform_SourceTransformFnServer interface { + Send(*SourceTransformResponse) error + Recv() (*SourceTransformRequest, error) + grpc.ServerStream +} + +type sourceTransformSourceTransformFnServer struct { + grpc.ServerStream +} + +func (x *sourceTransformSourceTransformFnServer) Send(m *SourceTransformResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *sourceTransformSourceTransformFnServer) Recv() (*SourceTransformRequest, error) { + m := new(SourceTransformRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { return nil, err } - if interceptor == nil { - return srv.(SourceTransformServer).SourceTransformFn(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/sourcetransformer.v1.SourceTransform/SourceTransformFn", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SourceTransformServer).SourceTransformFn(ctx, req.(*SourceTransformRequest)) - } - return interceptor(ctx, in, info, handler) + return m, nil } func _SourceTransform_IsReady_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { @@ -121,7 +158,7 @@ func _SourceTransform_IsReady_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/sourcetransformer.v1.SourceTransform/IsReady", + FullMethod: SourceTransform_IsReady_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(SourceTransformServer).IsReady(ctx, req.(*emptypb.Empty)) @@ -136,15 +173,18 @@ var SourceTransform_ServiceDesc = grpc.ServiceDesc{ ServiceName: "sourcetransformer.v1.SourceTransform", HandlerType: (*SourceTransformServer)(nil), Methods: []grpc.MethodDesc{ - { - MethodName: "SourceTransformFn", - Handler: _SourceTransform_SourceTransformFn_Handler, - }, { MethodName: "IsReady", Handler: _SourceTransform_IsReady_Handler, }, }, - Streams: []grpc.StreamDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "SourceTransformFn", + Handler: _SourceTransform_SourceTransformFn_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, Metadata: "pkg/apis/proto/sourcetransform/v1/transform.proto", } diff --git a/pkg/apis/proto/sourcetransform/v1/transformmock/transformmock.go b/pkg/apis/proto/sourcetransform/v1/transformmock/transformmock.go index fbc66555..8b347d06 100644 --- a/pkg/apis/proto/sourcetransform/v1/transformmock/transformmock.go +++ b/pkg/apis/proto/sourcetransform/v1/transformmock/transformmock.go @@ -58,21 +58,21 @@ func (mr *MockSourceTransformClientMockRecorder) IsReady(arg0, arg1 interface{}, } // SourceTransformFn mocks base method. -func (m *MockSourceTransformClient) SourceTransformFn(arg0 context.Context, arg1 *v1.SourceTransformRequest, arg2 ...grpc.CallOption) (*v1.SourceTransformResponse, error) { +func (m *MockSourceTransformClient) SourceTransformFn(arg0 context.Context, arg1 ...grpc.CallOption) (v1.SourceTransform_SourceTransformFnClient, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { + varargs := []interface{}{arg0} + for _, a := range arg1 { varargs = append(varargs, a) } ret := m.ctrl.Call(m, "SourceTransformFn", varargs...) - ret0, _ := ret[0].(*v1.SourceTransformResponse) + ret0, _ := ret[0].(v1.SourceTransform_SourceTransformFnClient) ret1, _ := ret[1].(error) return ret0, ret1 } // SourceTransformFn indicates an expected call of SourceTransformFn. -func (mr *MockSourceTransformClientMockRecorder) SourceTransformFn(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockSourceTransformClientMockRecorder) SourceTransformFn(arg0 interface{}, arg1 ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) + varargs := append([]interface{}{arg0}, arg1...) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SourceTransformFn", reflect.TypeOf((*MockSourceTransformClient)(nil).SourceTransformFn), varargs...) } diff --git a/pkg/batchmapper/examples/batchmap_flatmap/go.mod b/pkg/batchmapper/examples/batchmap_flatmap/go.mod index 99f7873e..cd04942a 100644 --- a/pkg/batchmapper/examples/batchmap_flatmap/go.mod +++ b/pkg/batchmapper/examples/batchmap_flatmap/go.mod @@ -1,19 +1,17 @@ module batchmap-flatmap -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - go.uber.org/atomic v1.11.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/batchmapper/examples/batchmap_flatmap/go.sum b/pkg/batchmapper/examples/batchmap_flatmap/go.sum index e884f1a5..36997a49 100644 --- a/pkg/batchmapper/examples/batchmap_flatmap/go.sum +++ b/pkg/batchmapper/examples/batchmap_flatmap/go.sum @@ -1,33 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/batchmapper/server.go b/pkg/batchmapper/server.go index 61736427..701aff63 100644 --- a/pkg/batchmapper/server.go +++ b/pkg/batchmapper/server.go @@ -3,6 +3,7 @@ package batchmapper import ( "context" "fmt" + "log" "os/signal" "sync" "syscall" @@ -10,7 +11,7 @@ import ( "google.golang.org/grpc" "github.com/numaproj/numaflow-go/pkg" - batchmappb "github.com/numaproj/numaflow-go/pkg/apis/proto/batchmap/v1" + mappb "github.com/numaproj/numaflow-go/pkg/apis/proto/map/v1" "github.com/numaproj/numaflow-go/pkg/info" "github.com/numaproj/numaflow-go/pkg/shared" ) @@ -66,7 +67,7 @@ func (m *server) Start(ctx context.Context) error { m.grpcServer = shared.CreateGRPCServer(m.opts.maxMessageSize) // register the batch map service - batchmappb.RegisterBatchMapServer(m.grpcServer, m.svc) + mappb.RegisterMapServer(m.grpcServer, m.svc) // start a go routine to stop the server gracefully when the context is done // or a shutdown signal is received from the service @@ -76,6 +77,7 @@ func (m *server) Start(ctx context.Context) error { defer wg.Done() select { case <-m.shutdownCh: + log.Printf("received shutdown signal") case <-ctxWithSignal.Done(): } shared.StopGRPCServer(m.grpcServer) diff --git a/pkg/batchmapper/service.go b/pkg/batchmapper/service.go index ba4df99d..676e2349 100644 --- a/pkg/batchmapper/service.go +++ b/pkg/batchmapper/service.go @@ -2,17 +2,16 @@ package batchmapper import ( "context" + "errors" "fmt" "io" "log" + "runtime/debug" - "go.uber.org/atomic" "golang.org/x/sync/errgroup" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" - batchmappb "github.com/numaproj/numaflow-go/pkg/apis/proto/batchmap/v1" + mappb "github.com/numaproj/numaflow-go/pkg/apis/proto/map/v1" ) const ( @@ -22,112 +21,185 @@ const ( serverInfoFilePath = "/var/run/numaflow/mapper-server-info" ) -// Service implements the proto gen server interface and contains the map operation -// handler. +// Service implements the proto gen server interface and contains the map operation handler. type Service struct { - batchmappb.UnimplementedBatchMapServer + mappb.UnimplementedMapServer BatchMapper BatchMapper shutdownCh chan<- struct{} } // IsReady returns true to indicate the gRPC connection is ready. -func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*batchmappb.ReadyResponse, error) { - return &batchmappb.ReadyResponse{Ready: true}, nil +func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*mappb.ReadyResponse, error) { + return &mappb.ReadyResponse{Ready: true}, nil } -// BatchMapFn applies a user defined function to a stream of request element and streams back the responses for them. -func (fs *Service) BatchMapFn(stream batchmappb.BatchMap_BatchMapFnServer) error { +// MapFn applies a user defined function to a stream of request elements and streams back the responses for them. +func (fs *Service) MapFn(stream mappb.Map_MapFnServer) error { ctx := stream.Context() - var g errgroup.Group - - // totalRequests is a counter for keeping a track of the number of datum requests - // that were received on the stream. We use an atomic int as this needs to be synchronized - // between the request/response go routines. - totalRequests := atomic.NewInt32(0) - - // datumStreamCh is used to stream messages to the user code interface - // As the BatchMap interface expects a list of request elements - // we read all the requests coming on the stream and keep streaming them to the user code on this channel. - datumStreamCh := make(chan Datum) - - // go routine to invoke the user handler function, and process the responses. - g.Go(func() error { - // handle panic - defer func() { - if r := recover(); r != nil { - fs.shutdownCh <- struct{}{} - } - }() - // Apply the user BatchMap implementation function - responses := fs.BatchMapper.BatchMap(ctx, datumStreamCh) - - // If the number of responses received does not align with the request batch size, - // we will not be able to process the data correctly. - // This should be marked as an error and the container is restarted. - // As this is a user error, we restart the container to mitigate any transient error otherwise, this - // crash should indicate to the user that there is some issue. - if len(responses.Items()) != int(totalRequests.Load()) { - errMsg := fmt.Sprintf("batchMapFn: mismatch between length of batch requests and responses, "+ - "expected:%d, got:%d", int(totalRequests.Load()), len(responses.Items())) - log.Panic(errMsg) - } - // iterate over the responses received and covert to the required proto format - for _, batchResp := range responses.Items() { - var elements []*batchmappb.BatchMapResponse_Result - for _, resp := range batchResp.Items() { - elements = append(elements, &batchmappb.BatchMapResponse_Result{ - Keys: resp.Keys(), - Value: resp.Value(), - Tags: resp.Tags(), - }) - } - singleRequestResp := &batchmappb.BatchMapResponse{ - Results: elements, - Id: batchResp.Id(), - } - // We stream back the result for a single request ID - // this would contain all the responses for that request. - err := stream.Send(singleRequestResp) - if err != nil { - log.Println("BatchMapFn: Got an error while Send() on stream", err) - return err + // Perform handshake before entering the main loop + if err := fs.performHandshake(stream); err != nil { + return err + } + + for { + datumStreamCh := make(chan Datum) + g, groupCtx := errgroup.WithContext(ctx) + + g.Go(func() error { + return fs.receiveRequests(groupCtx, stream, datumStreamCh) + }) + + g.Go(func() error { + return fs.processData(groupCtx, stream, datumStreamCh) + }) + + // Wait for the goroutines to finish + if err := g.Wait(); err != nil { + if errors.Is(err, io.EOF) { + log.Printf("Stopping the BatchMapFn") + return nil } + log.Printf("Stopping the BatchMapFn with err, %s", err) + fs.shutdownCh <- struct{}{} + return err } - return nil - }) + } +} + +// performHandshake performs the handshake with the client. +func (fs *Service) performHandshake(stream mappb.Map_MapFnServer) error { + req, err := stream.Recv() + if err != nil { + log.Printf("error receiving handshake from stream: %v", err) + return err + } + + if req.Handshake == nil || !req.Handshake.Sot { + return fmt.Errorf("expected handshake message") + } + + handshakeResponse := &mappb.MapResponse{ + Handshake: &mappb.Handshake{ + Sot: true, + }, + } + if err := stream.Send(handshakeResponse); err != nil { + return err + } + + return nil +} + +// recvWithContext wraps stream.Recv() to respect context cancellation. +func recvWithContext(ctx context.Context, stream mappb.Map_MapFnServer) (*mappb.MapRequest, error) { + type recvResult struct { + req *mappb.MapRequest + err error + } + + resultCh := make(chan recvResult, 1) + go func() { + req, err := stream.Recv() + resultCh <- recvResult{req: req, err: err} + }() + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case result := <-resultCh: + return result.req, result.err + } +} + +// receiveRequests receives the requests from the client and writes them to the datumStreamCh channel. +func (fs *Service) receiveRequests(ctx context.Context, stream mappb.Map_MapFnServer, datumStreamCh chan<- Datum) error { + defer close(datumStreamCh) - // loop to keep reading messages from the stream and sending it to the datumStreamCh for { - d, err := stream.Recv() - // if we see EOF on the stream we do not have any more messages coming up - if err == io.EOF { - // close the input data channel to indicate that no more messages expected - close(datumStreamCh) - break + req, err := recvWithContext(ctx, stream) + if errors.Is(err, context.Canceled) { + log.Printf("Context cancelled, stopping the MapBatchFn") + return err + } + if errors.Is(err, io.EOF) { + log.Printf("EOF received, stopping the MapBatchFn") + return err } if err != nil { - // close the input data channel to indicate that no more messages expected - close(datumStreamCh) - log.Println("BatchMapFn: Got an error while recv() on stream", err) + log.Printf("error receiving from batch map stream: %v", err) return err } - var hd = NewHandlerDatum(d.GetValue(), d.GetEventTime().AsTime(), d.GetWatermark().AsTime(), d.GetHeaders(), d.GetId(), d.GetKeys()) - // send the datum to the input channel - datumStreamCh <- hd - // Increase the counter for number of requests received - totalRequests.Inc() + + if req.Status != nil && req.Status.Eot { + break + } + + datum := &handlerDatum{ + id: req.GetId(), + value: req.GetRequest().GetValue(), + keys: req.GetRequest().GetKeys(), + eventTime: req.GetRequest().GetEventTime().AsTime(), + watermark: req.GetRequest().GetWatermark().AsTime(), + headers: req.GetRequest().GetHeaders(), + } + select { + case <-ctx.Done(): + return ctx.Err() + case datumStreamCh <- datum: + } } + return nil +} - // wait for all the responses to be processed - err := g.Wait() - // if there was any error during processing return the error - if err != nil { - statusErr := status.Errorf(codes.Internal, err.Error()) - return statusErr +// processData invokes the batch mapper to process the data and sends the response back to the client. +func (fs *Service) processData(ctx context.Context, stream mappb.Map_MapFnServer, datumStreamCh chan Datum) (err error) { + defer func() { + if r := recover(); r != nil { + log.Printf("panic inside batch map handler: %v %v", r, string(debug.Stack())) + err = fmt.Errorf("panic inside batch map handler: %v", r) + } + }() + + responses := fs.BatchMapper.BatchMap(ctx, datumStreamCh) + + for _, batchResp := range responses.Items() { + var elements []*mappb.MapResponse_Result + for _, resp := range batchResp.Items() { + elements = append(elements, &mappb.MapResponse_Result{ + Keys: resp.Keys(), + Value: resp.Value(), + Tags: resp.Tags(), + }) + } + singleRequestResp := &mappb.MapResponse{ + Results: elements, + Id: batchResp.Id(), + } + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + if err := stream.Send(singleRequestResp); err != nil { + log.Println("BatchMapFn: Got an error while Send() on stream", err) + return err + } } - // Once all responses are sent we can return, this would indicate the end of the rpc and - // send an EOF to the client on the stream + select { + case <-ctx.Done(): + return ctx.Err() + default: + // send the end of transmission message + eot := &mappb.MapResponse{ + Status: &mappb.TransmissionStatus{ + Eot: true, + }, + } + if err := stream.Send(eot); err != nil { + log.Println("BatchMapFn: Got an error while Send() on stream", err) + } + } return nil } diff --git a/pkg/batchmapper/service_test.go b/pkg/batchmapper/service_test.go index af387948..5184882c 100644 --- a/pkg/batchmapper/service_test.go +++ b/pkg/batchmapper/service_test.go @@ -13,20 +13,20 @@ import ( "google.golang.org/grpc" "google.golang.org/protobuf/types/known/timestamppb" - batchmappb "github.com/numaproj/numaflow-go/pkg/apis/proto/batchmap/v1" + mappb "github.com/numaproj/numaflow-go/pkg/apis/proto/map/v1" ) type BatchMapStreamFnServerTest struct { ctx context.Context - outputCh chan *batchmappb.BatchMapResponse - inputCh chan *batchmappb.BatchMapRequest + outputCh chan *mappb.MapResponse + inputCh chan *mappb.MapRequest grpc.ServerStream } func NewBatchBatchMapStreamFnServerTest( ctx context.Context, - inputCh chan *batchmappb.BatchMapRequest, - outputCh chan *batchmappb.BatchMapResponse, + inputCh chan *mappb.MapRequest, + outputCh chan *mappb.MapResponse, ) *BatchMapStreamFnServerTest { return &BatchMapStreamFnServerTest{ ctx: ctx, @@ -35,7 +35,7 @@ func NewBatchBatchMapStreamFnServerTest( } } -func (u *BatchMapStreamFnServerTest) Recv() (*batchmappb.BatchMapRequest, error) { +func (u *BatchMapStreamFnServerTest) Recv() (*mappb.MapRequest, error) { val, ok := <-u.inputCh if !ok { return val, io.EOF @@ -43,7 +43,7 @@ func (u *BatchMapStreamFnServerTest) Recv() (*batchmappb.BatchMapRequest, error) return val, nil } -func (u *BatchMapStreamFnServerTest) Send(d *batchmappb.BatchMapResponse) error { +func (u *BatchMapStreamFnServerTest) Send(d *mappb.MapResponse) error { u.outputCh <- d return nil } @@ -54,15 +54,15 @@ func (u *BatchMapStreamFnServerTest) Context() context.Context { type BatchMapFnServerErrTest struct { ctx context.Context - inputCh chan *batchmappb.BatchMapRequest - outputCh chan *batchmappb.BatchMapResponse + inputCh chan *mappb.MapRequest + outputCh chan *mappb.MapResponse grpc.ServerStream } func NewBatchMapFnServerErrTest( ctx context.Context, - inputCh chan *batchmappb.BatchMapRequest, - outputCh chan *batchmappb.BatchMapResponse, + inputCh chan *mappb.MapRequest, + outputCh chan *mappb.MapResponse, ) *BatchMapFnServerErrTest { return &BatchMapFnServerErrTest{ @@ -72,7 +72,7 @@ func NewBatchMapFnServerErrTest( } } -func (u *BatchMapFnServerErrTest) Recv() (*batchmappb.BatchMapRequest, error) { +func (u *BatchMapFnServerErrTest) Recv() (*mappb.MapRequest, error) { val, ok := <-u.inputCh if !ok { return val, io.EOF @@ -80,7 +80,7 @@ func (u *BatchMapFnServerErrTest) Recv() (*batchmappb.BatchMapRequest, error) { return val, nil } -func (u *BatchMapFnServerErrTest) Send(_ *batchmappb.BatchMapResponse) error { +func (u *BatchMapFnServerErrTest) Send(_ *mappb.MapResponse) error { return fmt.Errorf("send error") } @@ -92,8 +92,8 @@ func TestService_BatchMapFn(t *testing.T) { tests := []struct { name string handler BatchMapper - input []*batchmappb.BatchMapRequest - expected []*batchmappb.BatchMapResponse + input []*mappb.MapRequest + expected []*mappb.MapResponse expectedErr bool }{ { @@ -107,22 +107,45 @@ func TestService_BatchMapFn(t *testing.T) { } return batchResponses }), - input: []*batchmappb.BatchMapRequest{{ - Keys: []string{"client"}, - Value: []byte(`test1`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - Id: "test1", - }, { - Keys: []string{"client"}, - Value: []byte(`test2`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - Id: "test2", - }}, - expected: []*batchmappb.BatchMapResponse{ + input: []*mappb.MapRequest{ { - Results: []*batchmappb.BatchMapResponse_Result{ + Handshake: &mappb.Handshake{ + Sot: true, + }, + }, + { + Request: &mappb.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test1`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Id: "test1", + }, + { + Request: &mappb.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test2`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Id: "test2", + }, + { + Request: &mappb.MapRequest_Request{}, + Status: &mappb.TransmissionStatus{ + Eot: true, + }, + }, + }, + expected: []*mappb.MapResponse{ + { + Handshake: &mappb.Handshake{ + Sot: true, + }, + }, + { + Results: []*mappb.MapResponse_Result{ { Keys: []string{"client_test"}, Value: []byte(`test1`), @@ -131,7 +154,7 @@ func TestService_BatchMapFn(t *testing.T) { Id: "test1", }, { - Results: []*batchmappb.BatchMapResponse_Result{ + Results: []*mappb.MapResponse_Result{ { Keys: []string{"client_test"}, Value: []byte(`test2`), @@ -139,6 +162,11 @@ func TestService_BatchMapFn(t *testing.T) { }, Id: "test2", }, + { + Status: &mappb.TransmissionStatus{ + Eot: true, + }, + }, }, expectedErr: false, }, @@ -153,22 +181,39 @@ func TestService_BatchMapFn(t *testing.T) { } return batchResponses }), - input: []*batchmappb.BatchMapRequest{{ - Keys: []string{"client"}, - Value: []byte(`test1`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - Id: "test1", - }, { - Keys: []string{"client"}, - Value: []byte(`test2`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - Id: "test2", - }}, - expected: []*batchmappb.BatchMapResponse{ + input: []*mappb.MapRequest{ + { + Handshake: &mappb.Handshake{ + Sot: true, + }, + }, + { + Request: &mappb.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test1`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Id: "test1", + }, + { + Request: &mappb.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test2`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + Id: "test2", + }, + }, + expected: []*mappb.MapResponse{ + { + Handshake: &mappb.Handshake{ + Sot: true, + }, + }, { - Results: []*batchmappb.BatchMapResponse_Result{ + Results: []*mappb.MapResponse_Result{ { Keys: []string{"client_test"}, Value: []byte(`test1`), @@ -177,7 +222,7 @@ func TestService_BatchMapFn(t *testing.T) { Id: "test1", }, { - Results: []*batchmappb.BatchMapResponse_Result{ + Results: []*mappb.MapResponse_Result{ { Keys: []string{"client_test"}, Value: []byte(`test2`), @@ -185,6 +230,11 @@ func TestService_BatchMapFn(t *testing.T) { }, Id: "test2", }, + { + Status: &mappb.TransmissionStatus{ + Eot: true, + }, + }, }, expectedErr: true, }, @@ -199,11 +249,11 @@ func TestService_BatchMapFn(t *testing.T) { // instead of the regular outgoing context in the real gRPC connection. ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() - inputCh := make(chan *batchmappb.BatchMapRequest) - outputCh := make(chan *batchmappb.BatchMapResponse) - result := make([]*batchmappb.BatchMapResponse, 0) + inputCh := make(chan *mappb.MapRequest, 3) + outputCh := make(chan *mappb.MapResponse) + result := make([]*mappb.MapResponse, 0) - var udfBatchMapFnStream batchmappb.BatchMap_BatchMapFnServer + var udfBatchMapFnStream mappb.Map_MapFnServer if tt.expectedErr { udfBatchMapFnStream = NewBatchMapFnServerErrTest(ctx, inputCh, outputCh) } else { @@ -216,7 +266,7 @@ func TestService_BatchMapFn(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - err = fs.BatchMapFn(udfBatchMapFnStream) + err = fs.MapFn(udfBatchMapFnStream) close(outputCh) }() @@ -243,7 +293,6 @@ func TestService_BatchMapFn(t *testing.T) { if !reflect.DeepEqual(result, tt.expected) { t.Errorf("BatchMapFn() got = %v, want %v", result, tt.expected) } - }) } } diff --git a/pkg/info/server_info.go b/pkg/info/server_info.go index a38423d0..15b15bfb 100644 --- a/pkg/info/server_info.go +++ b/pkg/info/server_info.go @@ -112,8 +112,9 @@ func Read(opts ...Option) (*ServerInfo, error) { // GetDefaultServerInfo returns a ServerInfo object with the default fields populated for Go-SDK func GetDefaultServerInfo() *ServerInfo { - serverInfo := &ServerInfo{Protocol: UDS, Language: Go, MinimumNumaflowVersion: MinimumNumaflowVersion, - Version: getSDKVersion()} - return serverInfo - + return &ServerInfo{ + Protocol: UDS, + Language: Go, + Version: getSDKVersion(), + } } diff --git a/pkg/info/server_info_test.go b/pkg/info/server_info_test.go index ab553a88..683311ea 100644 --- a/pkg/info/server_info_test.go +++ b/pkg/info/server_info_test.go @@ -56,8 +56,8 @@ func Test_Read_Write(t *testing.T) { defer os.Remove(filepath) info := &ServerInfo{ Protocol: TCP, - Language: Java, - MinimumNumaflowVersion: MinimumNumaflowVersion, + Language: Go, + MinimumNumaflowVersion: "1.3.1-z", Version: "11", Metadata: map[string]string{"key1": "value1", "key2": "value2"}, } diff --git a/pkg/info/types.go b/pkg/info/types.go index 64c17a12..47b9de61 100644 --- a/pkg/info/types.go +++ b/pkg/info/types.go @@ -10,9 +10,23 @@ const ( type Language string const ( - Go Language = "go" - Python Language = "python" - Java Language = "java" + Go Language = "go" +) + +type ContainerType string + +// the string content matches the corresponding server info file name. +// DO NOT change it unless the server info file name is changed. +const ( + Sourcer ContainerType = "sourcer" + Sourcetransformer ContainerType = "sourcetransformer" + Sinker ContainerType = "sinker" + Mapper ContainerType = "mapper" + Reducer ContainerType = "reducer" + Reducestreamer ContainerType = "reducestreamer" + Sessionreducer ContainerType = "sessionreducer" + Sideinput ContainerType = "sideinput" + Fbsinker ContainerType = "fb-sinker" ) type MapMode string @@ -26,8 +40,20 @@ const ( // MapModeKey is the key used in the server info metadata map to indicate which map mode is enabled. const MapModeKey = "MAP_MODE" -// MinimumNumaflowVersion specifies the minimum Numaflow version required by the current SDK version -const MinimumNumaflowVersion = "1.3.0-rc1" +// MinimumNumaflowVersion is the minimum version of Numaflow required by the current SDK version +// To update this value, please follow the instructions for MINIMUM_NUMAFLOW_VERSION in +// https://github.com/numaproj/numaflow-rs/blob/main/src/shared.rs +var MinimumNumaflowVersion = map[ContainerType]string{ + Sourcer: "1.4.0-z", + Sourcetransformer: "1.4.0-z", + Sinker: "1.4.0-z", + Mapper: "1.4.0-z", + Reducestreamer: "1.4.0-z", + Reducer: "1.4.0-z", + Sessionreducer: "1.4.0-z", + Sideinput: "1.4.0-z", + Fbsinker: "1.4.0-z", +} // ServerInfo is the information about the server type ServerInfo struct { diff --git a/pkg/mapper/examples/even_odd/go.mod b/pkg/mapper/examples/even_odd/go.mod index dcde2eb1..85dc57e0 100644 --- a/pkg/mapper/examples/even_odd/go.mod +++ b/pkg/mapper/examples/even_odd/go.mod @@ -1,17 +1,17 @@ module even_odd -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/mapper/examples/even_odd/go.sum b/pkg/mapper/examples/even_odd/go.sum index 95c8479a..36997a49 100644 --- a/pkg/mapper/examples/even_odd/go.sum +++ b/pkg/mapper/examples/even_odd/go.sum @@ -1,24 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/mapper/examples/flatmap/go.mod b/pkg/mapper/examples/flatmap/go.mod index ffccc83a..c8d1fdfa 100644 --- a/pkg/mapper/examples/flatmap/go.mod +++ b/pkg/mapper/examples/flatmap/go.mod @@ -1,17 +1,17 @@ module flatmap -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/mapper/examples/flatmap/go.sum b/pkg/mapper/examples/flatmap/go.sum index 95c8479a..36997a49 100644 --- a/pkg/mapper/examples/flatmap/go.sum +++ b/pkg/mapper/examples/flatmap/go.sum @@ -1,24 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/mapper/examples/forward_message/go.mod b/pkg/mapper/examples/forward_message/go.mod index 641a20c5..89864ff3 100644 --- a/pkg/mapper/examples/forward_message/go.mod +++ b/pkg/mapper/examples/forward_message/go.mod @@ -1,17 +1,17 @@ module forward_message -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/mapper/examples/forward_message/go.sum b/pkg/mapper/examples/forward_message/go.sum index 95c8479a..36997a49 100644 --- a/pkg/mapper/examples/forward_message/go.sum +++ b/pkg/mapper/examples/forward_message/go.sum @@ -1,24 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/mapper/examples/retry/go.mod b/pkg/mapper/examples/retry/go.mod index fefcac61..4231789d 100644 --- a/pkg/mapper/examples/retry/go.mod +++ b/pkg/mapper/examples/retry/go.mod @@ -1,17 +1,17 @@ module retry -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/mapper/examples/retry/go.sum b/pkg/mapper/examples/retry/go.sum index 95c8479a..36997a49 100644 --- a/pkg/mapper/examples/retry/go.sum +++ b/pkg/mapper/examples/retry/go.sum @@ -1,24 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/mapper/examples/tickgen/go.mod b/pkg/mapper/examples/tickgen/go.mod index 6936777e..320a014d 100644 --- a/pkg/mapper/examples/tickgen/go.mod +++ b/pkg/mapper/examples/tickgen/go.mod @@ -1,17 +1,17 @@ module tickgen -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/mapper/examples/tickgen/go.sum b/pkg/mapper/examples/tickgen/go.sum index 95c8479a..36997a49 100644 --- a/pkg/mapper/examples/tickgen/go.sum +++ b/pkg/mapper/examples/tickgen/go.sum @@ -1,24 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/mapper/server.go b/pkg/mapper/server.go index 456f33c6..53f7dfb3 100644 --- a/pkg/mapper/server.go +++ b/pkg/mapper/server.go @@ -3,6 +3,7 @@ package mapper import ( "context" "fmt" + "log" "os/signal" "sync" "syscall" @@ -52,6 +53,7 @@ func (m *server) Start(ctx context.Context) error { // create a server info to the file, we need to add metadata to ensure selection of the // correct map mode, in this case unary map serverInfo := info.GetDefaultServerInfo() + serverInfo.MinimumNumaflowVersion = info.MinimumNumaflowVersion[info.Mapper] serverInfo.Metadata = map[string]string{info.MapModeKey: string(info.UnaryMap)} // start listening on unix domain socket @@ -76,6 +78,7 @@ func (m *server) Start(ctx context.Context) error { defer wg.Done() select { case <-m.shutdownCh: + log.Printf("shutdown signal received") case <-ctxWithSignal.Done(): } shared.StopGRPCServer(m.grpcServer) diff --git a/pkg/mapper/server_test.go b/pkg/mapper/server_test.go deleted file mode 100644 index 8c661ff0..00000000 --- a/pkg/mapper/server_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package mapper - -import ( - "context" - "os" - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" - - mappb "github.com/numaproj/numaflow-go/pkg/apis/proto/map/v1" -) - -func TestMapServer_Start(t *testing.T) { - socketFile, _ := os.CreateTemp("/tmp", "numaflow-test.sock") - defer func() { - _ = os.RemoveAll(socketFile.Name()) - }() - - serverInfoFile, _ := os.CreateTemp("/tmp", "numaflow-test-info") - defer func() { - _ = os.RemoveAll(serverInfoFile.Name()) - }() - - var mapHandler = MapperFunc(func(ctx context.Context, keys []string, d Datum) Messages { - msg := d.Value() - return MessagesBuilder().Append(NewMessage(msg).WithKeys([]string{keys[0] + "_test"})) - }) - // note: using actual uds connection - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) - defer cancel() - err := NewServer(mapHandler, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(serverInfoFile.Name())).Start(ctx) - assert.NoError(t, err) -} - -// tests the case where the server is shutdown gracefully when a panic occurs in the map handler -func TestMapServer_GracefulShutdown(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) - defer cancel() - - dir := t.TempDir() - socketFile, _ := os.Create(dir + "/test.sock") - defer func() { - _ = os.RemoveAll(socketFile.Name()) - }() - - serverInfoFile, _ := os.Create(dir + "/numaflow-test-info") - defer func() { - _ = os.RemoveAll(serverInfoFile.Name()) - }() - - var mapHandler = MapperFunc(func(ctx context.Context, keys []string, d Datum) Messages { - msg := d.Value() - if keys[0] == "key2" { - time.Sleep(20 * time.Millisecond) - panic("panic test") - } - time.Sleep(50 * time.Millisecond) - return MessagesBuilder().Append(NewMessage(msg).WithKeys([]string{keys[0] + "_test"})) - }) - - done := make(chan struct{}) - go func() { - err := NewServer(mapHandler, WithSockAddr(socketFile.Name()), WithServerInfoFilePath(socketFile.Name())).Start(ctx) - assert.NoError(t, err) - close(done) - }() - - // wait for the server to start - time.Sleep(10 * time.Millisecond) - - // create a client - conn, err := grpc.Dial( - "unix://"+socketFile.Name(), - grpc.WithTransportCredentials(insecure.NewCredentials()), - ) - if err != nil { - t.Fatalf("Failed to dial server: %v", err) - } - defer conn.Close() - - client := mappb.NewMapClient(conn) - // send two map requests with key1 and key2 as keys simultaneously - keys := []string{"key1", "key2"} - var wg sync.WaitGroup - for _, key := range keys { - wg.Add(1) - go func(key string) { - defer wg.Done() - req := &mappb.MapRequest{ - Keys: []string{key}, - } - - resp, err := client.MapFn(ctx, req) - // since there is a panic in the map handler for key2, we should get an error - // other requests should be successful - if key == "key2" { - assert.Error(t, err) - } else { - assert.NoError(t, err) - assert.NotNil(t, resp) - } - }(key) - } - - wg.Wait() - // wait for the server to shutdown gracefully because of the panic - select { - case <-ctx.Done(): - t.Fatal("server did not shutdown gracefully") - case <-done: - } -} diff --git a/pkg/mapper/service.go b/pkg/mapper/service.go index 7d5c2dc7..bc1cb64d 100644 --- a/pkg/mapper/service.go +++ b/pkg/mapper/service.go @@ -2,7 +2,13 @@ package mapper import ( "context" + "errors" + "fmt" + "io" + "log" + "runtime/debug" + "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" @@ -30,20 +36,134 @@ func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*mappb.ReadyRespons return &mappb.ReadyResponse{Ready: true}, nil } +// recvWithContext wraps stream.Recv() to respect context cancellation. +func recvWithContext(ctx context.Context, stream mappb.Map_MapFnServer) (*mappb.MapRequest, error) { + type recvResult struct { + req *mappb.MapRequest + err error + } + + resultCh := make(chan recvResult, 1) + go func() { + req, err := stream.Recv() + resultCh <- recvResult{req: req, err: err} + }() + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case result := <-resultCh: + return result.req, result.err + } +} + // MapFn applies a user defined function to each request element and returns a list of results. -func (fs *Service) MapFn(ctx context.Context, d *mappb.MapRequest) (_ *mappb.MapResponse, err error) { - var hd = NewHandlerDatum(d.GetValue(), d.GetEventTime().AsTime(), d.GetWatermark().AsTime(), d.GetHeaders()) - var elements []*mappb.MapResponse_Result +func (fs *Service) MapFn(stream mappb.Map_MapFnServer) error { + // perform handshake with client before processing requests + if err := fs.performHandshake(stream); err != nil { + return err + } + + ctx, cancel := context.WithCancel(stream.Context()) + defer cancel() + + // Use error group to manage goroutines, the groupCtx is cancelled when any of the + // goroutines return an error for the first time or the first time the wait returns. + g, groupCtx := errgroup.WithContext(ctx) + + // Channel to collect responses + responseCh := make(chan *mappb.MapResponse, 500) // FIXME: identify the right buffer size + defer close(responseCh) + + // Dedicated goroutine to send responses to the stream + g.Go(func() error { + for { + select { + case resp := <-responseCh: + if err := stream.Send(resp); err != nil { + log.Printf("Failed to send response: %v", err) + return err + } + case <-groupCtx.Done(): + return groupCtx.Err() + } + } + }) + + var readErr error + // Read requests from the stream and process them +outer: + for { + req, err := recvWithContext(groupCtx, stream) + if errors.Is(err, context.Canceled) { + log.Printf("Context cancelled, stopping the MapFn") + break outer + } + if errors.Is(err, io.EOF) { + log.Printf("EOF received, stopping the MapFn") + break outer + } + if err != nil { + log.Printf("Failed to receive request: %v", err) + readErr = err + // read loop is not part of the error group, so we need to cancel the context + // to signal the other goroutines to stop processing. + cancel() + break outer + } + g.Go(func() error { + return fs.handleRequest(groupCtx, req, responseCh) + }) + } + + // wait for all goroutines to finish + if err := g.Wait(); err != nil { + log.Printf("Stopping the MapFn with err, %s", err) + fs.shutdownCh <- struct{}{} + return status.Errorf(codes.Internal, "error processing requests: %v", err) + } + + // check if there was an error while reading from the stream + if readErr != nil { + return status.Errorf(codes.Internal, readErr.Error()) + } + + return nil +} - // Use defer and recover to handle panic +// performHandshake handles the handshake logic at the start of the stream. +func (fs *Service) performHandshake(stream mappb.Map_MapFnServer) error { + req, err := stream.Recv() + if err != nil { + return status.Errorf(codes.Internal, "failed to receive handshake: %v", err) + } + if req.GetHandshake() == nil || !req.GetHandshake().GetSot() { + return status.Errorf(codes.InvalidArgument, "invalid handshake") + } + handshakeResponse := &mappb.MapResponse{ + Handshake: &mappb.Handshake{ + Sot: true, + }, + } + if err := stream.Send(handshakeResponse); err != nil { + return fmt.Errorf("sending handshake response to client over gRPC stream: %w", err) + } + return nil +} + +// handleRequest processes each request and sends the response to the response channel. +func (fs *Service) handleRequest(ctx context.Context, req *mappb.MapRequest, responseCh chan<- *mappb.MapResponse) (err error) { defer func() { if r := recover(); r != nil { - fs.shutdownCh <- struct{}{} // Send shutdown signal - err = status.Errorf(codes.Internal, "panic occurred in Mapper.Map: %v", r) + log.Printf("panic inside map handler: %v %v", r, string(debug.Stack())) + err = status.Errorf(codes.Internal, "panic inside map handler: %v", r) } }() - messages := fs.Mapper.Map(ctx, d.GetKeys(), hd) + request := req.GetRequest() + hd := NewHandlerDatum(request.GetValue(), request.GetEventTime().AsTime(), request.GetWatermark().AsTime(), request.GetHeaders()) + messages := fs.Mapper.Map(ctx, request.GetKeys(), hd) + var elements []*mappb.MapResponse_Result for _, m := range messages.Items() { elements = append(elements, &mappb.MapResponse_Result{ Keys: m.Keys(), @@ -51,8 +171,14 @@ func (fs *Service) MapFn(ctx context.Context, d *mappb.MapRequest) (_ *mappb.Map Tags: m.Tags(), }) } - datumList := &mappb.MapResponse{ + resp := &mappb.MapResponse{ Results: elements, + Id: req.GetId(), + } + select { + case responseCh <- resp: + case <-ctx.Done(): + return ctx.Err() } - return datumList, err + return nil } diff --git a/pkg/mapper/service_test.go b/pkg/mapper/service_test.go index 48e5a290..7ec939d1 100644 --- a/pkg/mapper/service_test.go +++ b/pkg/mapper/service_test.go @@ -2,26 +2,82 @@ package mapper import ( "context" - "reflect" + "errors" + "fmt" + "net" "testing" "time" - "google.golang.org/protobuf/types/known/timestamppb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + + proto "github.com/numaproj/numaflow-go/pkg/apis/proto/map/v1" - mappb "github.com/numaproj/numaflow-go/pkg/apis/proto/map/v1" + "google.golang.org/protobuf/types/known/timestamppb" ) -func TestService_MapFn(t *testing.T) { +func newTestServer(t *testing.T, register func(server *grpc.Server)) *grpc.ClientConn { + lis := bufconn.Listen(1024 * 1024) + t.Cleanup(func() { + _ = lis.Close() + }) + + server := grpc.NewServer() + t.Cleanup(func() { + server.Stop() + }) + + register(server) + + errChan := make(chan error, 1) + go func() { + // t.Fatal should only be called from the goroutine running the test + if err := server.Serve(lis); err != nil { + errChan <- err + } + }() + + dialer := func(context.Context, string) (net.Conn, error) { + return lis.Dial() + } + + conn, err := grpc.NewClient("passthrough://", grpc.WithContextDialer(dialer), grpc.WithTransportCredentials(insecure.NewCredentials())) + t.Cleanup(func() { + _ = conn.Close() + }) + if err != nil { + t.Fatalf("Creating new gRPC client connection: %v", err) + } + + var grpcServerErr error + select { + case grpcServerErr = <-errChan: + case <-time.After(500 * time.Millisecond): + grpcServerErr = errors.New("gRPC server didn't start in 500ms") + } + if err != nil { + t.Fatalf("Failed to start gRPC server: %v", grpcServerErr) + } + + return conn +} + +func TestService_mapFn(t *testing.T) { type args struct { ctx context.Context - d *mappb.MapRequest + d *proto.MapRequest } + tests := []struct { name string handler Mapper args args - want *mappb.MapResponse - wantErr bool + want *proto.MapResponse }{ { name: "map_fn_forward_msg", @@ -31,22 +87,23 @@ func TestService_MapFn(t *testing.T) { }), args: args{ ctx: context.Background(), - d: &mappb.MapRequest{ - Keys: []string{"client"}, - Value: []byte(`test`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + d: &proto.MapRequest{ + Request: &proto.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, }, }, - want: &mappb.MapResponse{ - Results: []*mappb.MapResponse_Result{ + want: &proto.MapResponse{ + Results: []*proto.MapResponse_Result{ { Keys: []string{"client_test"}, Value: []byte(`test`), }, }, }, - wantErr: false, }, { name: "map_fn_forward_msg_forward_to_all", @@ -56,21 +113,22 @@ func TestService_MapFn(t *testing.T) { }), args: args{ ctx: context.Background(), - d: &mappb.MapRequest{ - Keys: []string{"client"}, - Value: []byte(`test`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + d: &proto.MapRequest{ + Request: &proto.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, }, }, - want: &mappb.MapResponse{ - Results: []*mappb.MapResponse_Result{ + want: &proto.MapResponse{ + Results: []*proto.MapResponse_Result{ { Value: []byte(`test`), }, }, }, - wantErr: false, }, { name: "map_fn_forward_msg_drop_msg", @@ -79,41 +137,157 @@ func TestService_MapFn(t *testing.T) { }), args: args{ ctx: context.Background(), - d: &mappb.MapRequest{ - Keys: []string{"client"}, - Value: []byte(`test`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + d: &proto.MapRequest{ + Request: &proto.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, }, }, - want: &mappb.MapResponse{ - Results: []*mappb.MapResponse_Result{ + want: &proto.MapResponse{ + Results: []*proto.MapResponse_Result{ { Tags: []string{DROP}, - Value: []byte{}, + Value: nil, }, }, }, - wantErr: false, }, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fs := &Service{ + svc := &Service{ Mapper: tt.handler, } - // here's a trick for testing: - // because we are not using gRPC, we directly set a new incoming ctx - // instead of the regular outgoing context in the real gRPC connection. - ctx := context.Background() - got, err := fs.MapFn(ctx, tt.args.d) - if (err != nil) != tt.wantErr { - t.Errorf("MapFn() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("MapFn() got = %v, want %v", got, tt.want) - } + + conn := newTestServer(t, func(server *grpc.Server) { + proto.RegisterMapServer(server, svc) + }) + + client := proto.NewMapClient(conn) + stream, err := client.MapFn(context.Background()) + require.NoError(t, err, "Creating stream") + + doHandshake(t, stream) + + err = stream.Send(tt.args.d) + require.NoError(t, err, "Sending message over the stream") + + got, err := stream.Recv() + require.NoError(t, err, "Receiving message from the stream") + + assert.Equal(t, got.Results, tt.want.Results) }) } } + +func doHandshake(t *testing.T, stream proto.Map_MapFnClient) { + t.Helper() + handshakeReq := &proto.MapRequest{ + Handshake: &proto.Handshake{Sot: true}, + } + err := stream.Send(handshakeReq) + require.NoError(t, err, "Sending handshake request to the stream") + + handshakeResp, err := stream.Recv() + require.NoError(t, err, "Receiving handshake response") + + require.Empty(t, handshakeResp.Results, "Invalid handshake response") + require.Empty(t, handshakeResp.Id, "Invalid handshake response") + require.NotNil(t, handshakeResp.Handshake, "Invalid handshake response") + require.True(t, handshakeResp.Handshake.Sot, "Invalid handshake response") +} + +func TestService_MapFn_Multiple_Messages(t *testing.T) { + svc := &Service{ + Mapper: MapperFunc(func(ctx context.Context, keys []string, datum Datum) Messages { + msg := datum.Value() + return MessagesBuilder().Append(NewMessage(msg).WithKeys([]string{keys[0] + "_test"})) + }), + } + conn := newTestServer(t, func(server *grpc.Server) { + proto.RegisterMapServer(server, svc) + }) + + client := proto.NewMapClient(conn) + stream, err := client.MapFn(context.Background()) + require.NoError(t, err, "Creating stream") + + doHandshake(t, stream) + + const msgCount = 10 + for i := 0; i < msgCount; i++ { + msg := proto.MapRequest{ + Request: &proto.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(fmt.Sprintf("test_%d", i)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + } + err = stream.Send(&msg) + require.NoError(t, err, "Sending message over the stream") + } + err = stream.CloseSend() + require.NoError(t, err, "Closing the send direction of the stream") + + expectedResults := make([][]*proto.MapResponse_Result, msgCount) + for i := 0; i < msgCount; i++ { + expectedResults[i] = []*proto.MapResponse_Result{ + { + Keys: []string{"client_test"}, + Value: []byte(fmt.Sprintf("test_%d", i)), + }, + } + } + + results := make([][]*proto.MapResponse_Result, msgCount) + for i := 0; i < msgCount; i++ { + got, err := stream.Recv() + require.NoError(t, err, "Receiving message from the stream") + results[i] = got.Results + } + require.ElementsMatch(t, results, expectedResults) +} + +func TestService_MapFn_Panic(t *testing.T) { + svc := &Service{ + Mapper: MapperFunc(func(ctx context.Context, keys []string, datum Datum) Messages { + panic("map failed") + }), + // panic in the transformer causes the server to send a shutdown signal to shutdownCh channel. + // The function that errgroup runs in a goroutine will be blocked until this shutdown signal is received somewhere else. + // Since we don't listen for shutdown signal in the tests, we use buffered channel to unblock the server function. + shutdownCh: make(chan<- struct{}, 1), + } + conn := newTestServer(t, func(server *grpc.Server) { + proto.RegisterMapServer(server, svc) + }) + + client := proto.NewMapClient(conn) + stream, err := client.MapFn(context.Background()) + require.NoError(t, err, "Creating stream") + + doHandshake(t, stream) + + msg := proto.MapRequest{ + Request: &proto.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte("test"), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + } + err = stream.Send(&msg) + require.NoError(t, err, "Sending message over the stream") + err = stream.CloseSend() + require.NoError(t, err, "Closing the send direction of the stream") + _, err = stream.Recv() + require.Error(t, err, "Expected error while receiving message from the stream") + gotStatus, _ := status.FromError(err) + expectedStatus := status.Convert(status.Errorf(codes.Internal, "error processing requests: rpc error: code = Internal desc = panic inside map handler: map failed")) + require.Equal(t, expectedStatus, gotStatus) +} diff --git a/pkg/mapstreamer/examples/flatmap_stream/go.mod b/pkg/mapstreamer/examples/flatmap_stream/go.mod index 946573ed..1b070f10 100644 --- a/pkg/mapstreamer/examples/flatmap_stream/go.mod +++ b/pkg/mapstreamer/examples/flatmap_stream/go.mod @@ -1,17 +1,17 @@ module flatmap_stream -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/mapstreamer/examples/flatmap_stream/go.sum b/pkg/mapstreamer/examples/flatmap_stream/go.sum index 95c8479a..36997a49 100644 --- a/pkg/mapstreamer/examples/flatmap_stream/go.sum +++ b/pkg/mapstreamer/examples/flatmap_stream/go.sum @@ -1,24 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/mapstreamer/examples/flatmap_stream/main.go b/pkg/mapstreamer/examples/flatmap_stream/main.go index a72ab3f3..151b13cc 100644 --- a/pkg/mapstreamer/examples/flatmap_stream/main.go +++ b/pkg/mapstreamer/examples/flatmap_stream/main.go @@ -13,6 +13,7 @@ type FlatMap struct { } func (f *FlatMap) MapStream(ctx context.Context, keys []string, d mapstreamer.Datum, messageCh chan<- mapstreamer.Message) { + // we have to close to indicate the end of the stream, otherwise the client will wait forever. defer close(messageCh) msg := d.Value() _ = d.EventTime() // Event time is available diff --git a/pkg/mapstreamer/server.go b/pkg/mapstreamer/server.go index 5b810fc9..42bdcd82 100644 --- a/pkg/mapstreamer/server.go +++ b/pkg/mapstreamer/server.go @@ -3,6 +3,7 @@ package mapstreamer import ( "context" "fmt" + "log" "os/signal" "sync" "syscall" @@ -10,7 +11,7 @@ import ( "google.golang.org/grpc" "github.com/numaproj/numaflow-go/pkg" - mapstreampb "github.com/numaproj/numaflow-go/pkg/apis/proto/mapstream/v1" + mappb "github.com/numaproj/numaflow-go/pkg/apis/proto/map/v1" "github.com/numaproj/numaflow-go/pkg/info" "github.com/numaproj/numaflow-go/pkg/shared" ) @@ -52,6 +53,7 @@ func (m *server) Start(ctx context.Context) error { // create a server info to the file, we need to add metadata to ensure selection of the // correct map mode, in this case streaming map serverInfo := info.GetDefaultServerInfo() + serverInfo.MinimumNumaflowVersion = info.MinimumNumaflowVersion[info.Mapper] serverInfo.Metadata = map[string]string{info.MapModeKey: string(info.StreamMap)} // start listening on unix domain socket @@ -66,7 +68,7 @@ func (m *server) Start(ctx context.Context) error { m.grpcServer = shared.CreateGRPCServer(m.opts.maxMessageSize) // register the map stream service - mapstreampb.RegisterMapStreamServer(m.grpcServer, m.svc) + mappb.RegisterMapServer(m.grpcServer, m.svc) // start a go routine to stop the server gracefully when the context is done // or a shutdown signal is received from the service @@ -76,6 +78,7 @@ func (m *server) Start(ctx context.Context) error { defer wg.Done() select { case <-m.shutdownCh: + log.Printf("shutdown signal received") case <-ctxWithSignal.Done(): } shared.StopGRPCServer(m.grpcServer) diff --git a/pkg/mapstreamer/service.go b/pkg/mapstreamer/service.go index 3a865869..8795ff08 100644 --- a/pkg/mapstreamer/service.go +++ b/pkg/mapstreamer/service.go @@ -2,10 +2,18 @@ package mapstreamer import ( "context" + "errors" + "fmt" + "io" + "log" + "runtime/debug" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" - mapstreampb "github.com/numaproj/numaflow-go/pkg/apis/proto/mapstream/v1" + mappb "github.com/numaproj/numaflow-go/pkg/apis/proto/map/v1" ) const ( @@ -18,62 +26,168 @@ const ( // Service implements the proto gen server interface and contains the map // streaming function. type Service struct { - mapstreampb.UnimplementedMapStreamServer + mappb.UnimplementedMapServer shutdownCh chan<- struct{} MapperStream MapStreamer } // IsReady returns true to indicate the gRPC connection is ready. -func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*mapstreampb.ReadyResponse, error) { - return &mapstreampb.ReadyResponse{Ready: true}, nil +func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*mappb.ReadyResponse, error) { + return &mappb.ReadyResponse{Ready: true}, nil } -// MapStreamFn applies a function to each request element and streams the results back. -func (fs *Service) MapStreamFn(d *mapstreampb.MapStreamRequest, stream mapstreampb.MapStream_MapStreamFnServer) error { - var hd = NewHandlerDatum(d.GetValue(), d.GetEventTime().AsTime(), d.GetWatermark().AsTime(), d.GetHeaders()) - ctx := stream.Context() - messageCh := make(chan Message) +// recvWithContext wraps stream.Recv() to respect context cancellation. +func recvWithContext(ctx context.Context, stream mappb.Map_MapFnServer) (*mappb.MapRequest, error) { + type recvResult struct { + req *mappb.MapRequest + err error + } - done := make(chan bool) + resultCh := make(chan recvResult, 1) go func() { - // handle panic - defer func() { - if r := recover(); r != nil { - fs.shutdownCh <- struct{}{} - } - }() - fs.MapperStream.MapStream(ctx, d.GetKeys(), hd, messageCh) - done <- true + req, err := stream.Recv() + resultCh <- recvResult{req: req, err: err} }() - finished := false + select { + case <-ctx.Done(): + return nil, ctx.Err() + case result := <-resultCh: + return result.req, result.err + } +} + +// MapFn applies a function to each request element and streams the results back. +func (fs *Service) MapFn(stream mappb.Map_MapFnServer) error { + // perform handshake with client before processing requests + if err := fs.performHandshake(stream); err != nil { + return err + } + + ctx, cancel := context.WithCancel(stream.Context()) + defer cancel() + + // Use error group to manage goroutines, the groupCtx is cancelled when any of the + // goroutines return an error for the first time or the first time the wait returns. + g, groupCtx := errgroup.WithContext(ctx) + var readErr error + // Read requests from the stream and process them +outer: + for { + req, err := recvWithContext(groupCtx, stream) + if errors.Is(err, context.Canceled) { + log.Printf("Context cancelled, stopping the MapFn") + break outer + } + if errors.Is(err, io.EOF) { + log.Printf("EOF received, stopping the MapFn") + break outer + } + if err != nil { + log.Printf("Failed to receive request: %v", err) + readErr = err + // read loop is not part of the error group, so we need to cancel the context + // to signal the other goroutines to stop processing. + cancel() + break outer + } + g.Go(func() error { + messageCh := make(chan Message) + workerGroup, innerCtx := errgroup.WithContext(groupCtx) + + workerGroup.Go(func() error { + return fs.invokeHandler(innerCtx, req, messageCh) + }) + + workerGroup.Go(func() error { + return fs.writeResponseToClient(innerCtx, stream, req.GetId(), messageCh) + }) + + return workerGroup.Wait() + }) + } + + // wait for all goroutines to finish + if err := g.Wait(); err != nil { + log.Printf("Stopping the MapFn with err, %s", err) + fs.shutdownCh <- struct{}{} + return status.Errorf(codes.Internal, "error processing requests: %v", err) + } + + // check if there was an error while reading from the stream + if readErr != nil { + return status.Errorf(codes.Internal, readErr.Error()) + } + + return nil +} + +// invokeHandler handles the map stream invocation. +func (fs *Service) invokeHandler(ctx context.Context, req *mappb.MapRequest, messageCh chan<- Message) (err error) { + defer func() { + if r := recover(); r != nil { + log.Printf("panic inside mapStream handler: %v %v", r, string(debug.Stack())) + err = fmt.Errorf("panic inside mapStream handler: %v", r) + return + } + }() + streamReq := req.GetRequest() + hd := NewHandlerDatum(streamReq.GetValue(), streamReq.GetEventTime().AsTime(), streamReq.GetWatermark().AsTime(), streamReq.GetHeaders()) + fs.MapperStream.MapStream(ctx, req.GetRequest().GetKeys(), hd, messageCh) + return nil +} + +// writeResponseToClient writes the response back to the client. +func (fs *Service) writeResponseToClient(ctx context.Context, stream mappb.Map_MapFnServer, reqID string, messageCh <-chan Message) error { for { select { - case <-done: - finished = true case message, ok := <-messageCh: if !ok { - // Channel already closed, not closing again. + // Send EOT message since we are done processing the request. + eotMessage := &mappb.MapResponse{ + Status: &mappb.TransmissionStatus{Eot: true}, + Id: reqID, + } + if err := stream.Send(eotMessage); err != nil { + return err + } return nil } - element := &mapstreampb.MapStreamResponse{ - Result: &mapstreampb.MapStreamResponse_Result{ - Keys: message.Keys(), - Value: message.Value(), - Tags: message.Tags(), + element := &mappb.MapResponse{ + Results: []*mappb.MapResponse_Result{ + { + Keys: message.Keys(), + Value: message.Value(), + Tags: message.Tags(), + }, }, + Id: reqID, } - err := stream.Send(element) - // the error here is returned by stream.Send() which is already a gRPC error - if err != nil { - // Channel may or may not be closed, as we are not sure leave it to GC. + if err := stream.Send(element); err != nil { return err } - default: - if finished { - close(messageCh) - return nil - } + case <-ctx.Done(): + return ctx.Err() } } } + +// performHandshake handles the handshake logic at the start of the stream. +func (fs *Service) performHandshake(stream mappb.Map_MapFnServer) error { + req, err := stream.Recv() + if err != nil { + return status.Errorf(codes.Internal, "failed to receive handshake: %v", err) + } + if req.GetHandshake() == nil || !req.GetHandshake().GetSot() { + return status.Errorf(codes.InvalidArgument, "invalid handshake") + } + handshakeResponse := &mappb.MapResponse{ + Handshake: &mappb.Handshake{ + Sot: true, + }, + } + if err := stream.Send(handshakeResponse); err != nil { + return fmt.Errorf("sending handshake response to client over gRPC stream: %w", err) + } + return nil +} diff --git a/pkg/mapstreamer/service_test.go b/pkg/mapstreamer/service_test.go index 6b8f00a4..fd5b35e9 100644 --- a/pkg/mapstreamer/service_test.go +++ b/pkg/mapstreamer/service_test.go @@ -2,229 +2,431 @@ package mapstreamer import ( "context" + "errors" "fmt" - "reflect" - "sync" + "io" + "net" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "google.golang.org/grpc" - "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + + proto "github.com/numaproj/numaflow-go/pkg/apis/proto/map/v1" - mapstreampb "github.com/numaproj/numaflow-go/pkg/apis/proto/mapstream/v1" + "google.golang.org/protobuf/types/known/timestamppb" ) -type MapStreamFnServerTest struct { - ctx context.Context - outputCh chan mapstreampb.MapStreamResponse - grpc.ServerStream -} +func newTestServer(t *testing.T, register func(server *grpc.Server)) *grpc.ClientConn { + lis := bufconn.Listen(1024 * 1024) + t.Cleanup(func() { + _ = lis.Close() + }) -func NewMapStreamFnServerTest( - ctx context.Context, - outputCh chan mapstreampb.MapStreamResponse, -) *MapStreamFnServerTest { - return &MapStreamFnServerTest{ - ctx: ctx, - outputCh: outputCh, - } -} + server := grpc.NewServer() + t.Cleanup(func() { + server.Stop() + }) -func (u *MapStreamFnServerTest) Send(d *mapstreampb.MapStreamResponse) error { - u.outputCh <- *d - return nil -} + register(server) -func (u *MapStreamFnServerTest) Context() context.Context { - return u.ctx -} + errChan := make(chan error, 1) + go func() { + // t.Fatal should only be called from the goroutine running the test + if err := server.Serve(lis); err != nil { + errChan <- err + } + }() -type MapStreamFnServerErrTest struct { - ctx context.Context - grpc.ServerStream -} + dialer := func(context.Context, string) (net.Conn, error) { + return lis.Dial() + } -func NewMapStreamFnServerErrTest( - ctx context.Context, + conn, err := grpc.NewClient("passthrough://", grpc.WithContextDialer(dialer), grpc.WithTransportCredentials(insecure.NewCredentials())) + t.Cleanup(func() { + _ = conn.Close() + }) + if err != nil { + t.Fatalf("Creating new gRPC client connection: %v", err) + } -) *MapStreamFnServerErrTest { - return &MapStreamFnServerErrTest{ - ctx: ctx, + var grpcServerErr error + select { + case grpcServerErr = <-errChan: + case <-time.After(500 * time.Millisecond): + grpcServerErr = errors.New("gRPC server didn't start in 500ms") + } + if err != nil { + t.Fatalf("Failed to start gRPC server: %v", grpcServerErr) } -} -func (u *MapStreamFnServerErrTest) Send(_ *mapstreampb.MapStreamResponse) error { - return fmt.Errorf("send error") + return conn } -func (u *MapStreamFnServerErrTest) Context() context.Context { - return u.ctx -} +func TestService_MapFn(t *testing.T) { + type args struct { + ctx context.Context + d *proto.MapRequest + } -func TestService_MapFnStream(t *testing.T) { tests := []struct { - name string - handler MapStreamer - input *mapstreampb.MapStreamRequest - expected []*mapstreampb.MapStreamResponse - expectedErr bool - streamErr bool + name string + handler MapStreamer + args args + want *proto.MapResponse }{ { - name: "map_stream_fn_forward_msg", + name: "map_fn_forward_msg", handler: MapStreamerFunc(func(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) { + defer close(messageCh) msg := datum.Value() messageCh <- NewMessage(msg).WithKeys([]string{keys[0] + "_test"}) - close(messageCh) }), - input: &mapstreampb.MapStreamRequest{ - Keys: []string{"client"}, - Value: []byte(`test`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - }, - expected: []*mapstreampb.MapStreamResponse{ - { - Result: &mapstreampb.MapStreamResponse_Result{ - Keys: []string{"client_test"}, - Value: []byte(`test`), + args: args{ + ctx: context.Background(), + d: &proto.MapRequest{ + Request: &proto.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), }, }, }, - expectedErr: false, - }, - { - name: "map_stream_fn_forward_msg_without_close_stream", - handler: MapStreamerFunc(func(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) { - msg := datum.Value() - messageCh <- NewMessage(msg).WithKeys([]string{keys[0] + "_test"}) - }), - input: &mapstreampb.MapStreamRequest{ - Keys: []string{"client"}, - Value: []byte(`test`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - }, - expected: []*mapstreampb.MapStreamResponse{ - { - Result: &mapstreampb.MapStreamResponse_Result{ + want: &proto.MapResponse{ + Results: []*proto.MapResponse_Result{ + { Keys: []string{"client_test"}, Value: []byte(`test`), }, }, }, - expectedErr: false, }, { - name: "map_stream_fn_forward_msg_forward_to_all", + name: "map_fn_forward_msg_forward_to_all", handler: MapStreamerFunc(func(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) { + defer close(messageCh) msg := datum.Value() messageCh <- NewMessage(msg) - close(messageCh) }), - input: &mapstreampb.MapStreamRequest{ - Keys: []string{"client"}, - Value: []byte(`test`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + args: args{ + ctx: context.Background(), + d: &proto.MapRequest{ + Request: &proto.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + }, }, - expected: []*mapstreampb.MapStreamResponse{ - { - Result: &mapstreampb.MapStreamResponse_Result{ + want: &proto.MapResponse{ + Results: []*proto.MapResponse_Result{ + { Value: []byte(`test`), }, }, }, - expectedErr: false, }, { - name: "map_stream_fn_forward_msg_drop_msg", + name: "map_fn_forward_msg_drop_msg", handler: MapStreamerFunc(func(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) { + defer close(messageCh) messageCh <- MessageToDrop() - close(messageCh) }), - input: &mapstreampb.MapStreamRequest{ - Keys: []string{"client"}, - Value: []byte(`test`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - }, - expected: []*mapstreampb.MapStreamResponse{ - { - Result: &mapstreampb.MapStreamResponse_Result{ - Tags: []string{DROP}, - Value: []byte{}, + args: args{ + ctx: context.Background(), + d: &proto.MapRequest{ + Request: &proto.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), }, }, }, - expectedErr: false, - }, - { - name: "map_stream_fn_forward_err", - handler: MapStreamerFunc(func(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) { - messageCh <- MessageToDrop() - close(messageCh) - }), - input: &mapstreampb.MapStreamRequest{ - Keys: []string{"client"}, - Value: []byte(`test`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - }, - expected: []*mapstreampb.MapStreamResponse{ - { - Result: &mapstreampb.MapStreamResponse_Result{ + want: &proto.MapResponse{ + Results: []*proto.MapResponse_Result{ + { Tags: []string{DROP}, - Value: []byte{}, + Value: nil, }, }, }, - expectedErr: true, }, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fs := &Service{ + svc := &Service{ MapperStream: tt.handler, } - // here's a trick for testing: - // because we are not using gRPC, we directly set a new incoming ctx - // instead of the regular outgoing context in the real gRPC connection. - ctx := context.Background() - outputCh := make(chan mapstreampb.MapStreamResponse) - result := make([]*mapstreampb.MapStreamResponse, 0) - - var udfMapStreamFnStream mapstreampb.MapStream_MapStreamFnServer - if tt.streamErr { - udfMapStreamFnStream = NewMapStreamFnServerErrTest(ctx) - } else { - udfMapStreamFnStream = NewMapStreamFnServerTest(ctx, outputCh) - } - var wg sync.WaitGroup + conn := newTestServer(t, func(server *grpc.Server) { + proto.RegisterMapServer(server, svc) + }) - wg.Add(1) - go func() { - defer wg.Done() - for msg := range outputCh { - result = append(result, &msg) - } - }() + client := proto.NewMapClient(conn) + stream, err := client.MapFn(context.Background()) + require.NoError(t, err, "Creating stream") - err := fs.MapStreamFn(tt.input, udfMapStreamFnStream) - close(outputCh) - wg.Wait() + doHandshake(t, stream) + + err = stream.Send(tt.args.d) + require.NoError(t, err, "Sending message over the stream") + + got, err := stream.Recv() + require.NoError(t, err, "Receiving message from the stream") + + assert.Equal(t, got.Results, tt.want.Results) + }) + } +} + +func doHandshake(t *testing.T, stream proto.Map_MapFnClient) { + t.Helper() + handshakeReq := &proto.MapRequest{ + Handshake: &proto.Handshake{Sot: true}, + } + err := stream.Send(handshakeReq) + require.NoError(t, err, "Sending handshake request to the stream") - if err != nil { - assert.True(t, tt.expectedErr, "MapStreamFn() error = %v, expectedErr %v", err, tt.expectedErr) - return + handshakeResp, err := stream.Recv() + require.NoError(t, err, "Receiving handshake response") + + require.Empty(t, handshakeResp.Results, "Invalid handshake response") + require.Empty(t, handshakeResp.Id, "Invalid handshake response") + require.NotNil(t, handshakeResp.Handshake, "Invalid handshake response") + require.True(t, handshakeResp.Handshake.Sot, "Invalid handshake response") +} + +func TestService_MapFn_SingleMessage_MultipleResponses(t *testing.T) { + svc := &Service{ + MapperStream: MapStreamerFunc(func(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) { + defer close(messageCh) + for i := 0; i < 10; i++ { + msg := fmt.Sprintf("response_%d", i) + messageCh <- NewMessage([]byte(msg)).WithKeys([]string{keys[0] + "_test"}) } + }), + } + conn := newTestServer(t, func(server *grpc.Server) { + proto.RegisterMapServer(server, svc) + }) + + client := proto.NewMapClient(conn) + stream, err := client.MapFn(context.Background()) + require.NoError(t, err, "Creating stream") + + doHandshake(t, stream) + + msg := proto.MapRequest{ + Request: &proto.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte("test"), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + } + err = stream.Send(&msg) + require.NoError(t, err, "Sending message over the stream") + err = stream.CloseSend() + require.NoError(t, err, "Closing the send direction of the stream") + + expectedResults := make([][]*proto.MapResponse_Result, 10) + for i := 0; i < 10; i++ { + expectedResults[i] = []*proto.MapResponse_Result{ + { + Keys: []string{"client_test"}, + Value: []byte(fmt.Sprintf("response_%d", i)), + }, + } + } + + results := make([][]*proto.MapResponse_Result, 0) + for i := 0; i < 10; i++ { + got, err := stream.Recv() + require.NoError(t, err, "Receiving message from the stream") + results = append(results, got.Results) + } + + require.ElementsMatch(t, results, expectedResults) + +} + +func TestService_MapFn_Multiple_Messages(t *testing.T) { + svc := &Service{ + MapperStream: MapStreamerFunc(func(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) { + defer close(messageCh) + msg := datum.Value() + messageCh <- NewMessage(msg).WithKeys([]string{keys[0] + "_test"}) + }), + } + conn := newTestServer(t, func(server *grpc.Server) { + proto.RegisterMapServer(server, svc) + }) + + client := proto.NewMapClient(conn) + stream, err := client.MapFn(context.Background()) + require.NoError(t, err, "Creating stream") + + doHandshake(t, stream) + + const msgCount = 10 + for i := 0; i < msgCount; i++ { + msg := proto.MapRequest{ + Request: &proto.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(fmt.Sprintf("test_%d", i)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + } + err = stream.Send(&msg) + require.NoError(t, err, "Sending message over the stream") + } + err = stream.CloseSend() + require.NoError(t, err, "Closing the send direction of the stream") + + expectedResults := make([][]*proto.MapResponse_Result, msgCount) + results := make([][]*proto.MapResponse_Result, 0) + eotCount := 0 + + for { + got, err := stream.Recv() + if err == io.EOF { + break + } + require.NoError(t, err, "Receiving message from the stream") + + if got.Status != nil && got.Status.Eot { + eotCount++ + } else { + results = append(results, got.Results) + } + } + + require.Equal(t, msgCount, eotCount, "Expected number of EOT messages") - if !reflect.DeepEqual(result, tt.expected) { - t.Errorf("MapStreamFn() got = %v, want %v", result, tt.expected) + for i := 0; i < msgCount; i++ { + expectedResults[i] = []*proto.MapResponse_Result{ + { + Keys: []string{"client_test"}, + Value: []byte(fmt.Sprintf("test_%d", i)), + }, + } + } + + require.ElementsMatch(t, results, expectedResults) +} + +func TestService_MapFn_Panic(t *testing.T) { + svc := &Service{ + MapperStream: MapStreamerFunc(func(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) { + panic("map failed") + }), + shutdownCh: make(chan<- struct{}, 1), + } + conn := newTestServer(t, func(server *grpc.Server) { + proto.RegisterMapServer(server, svc) + }) + + client := proto.NewMapClient(conn) + stream, err := client.MapFn(context.Background()) + require.NoError(t, err, "Creating stream") + + doHandshake(t, stream) + + msg := proto.MapRequest{ + Request: &proto.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte("test"), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + } + err = stream.Send(&msg) + require.NoError(t, err, "Sending message over the stream") + err = stream.CloseSend() + require.NoError(t, err, "Closing the send direction of the stream") + _, err = stream.Recv() + require.Error(t, err, "Expected error while receiving message from the stream") + gotStatus, _ := status.FromError(err) + expectedStatus := status.Convert(status.Errorf(codes.Internal, "error processing requests: panic inside mapStream handler: map failed")) + require.Equal(t, expectedStatus, gotStatus) +} + +func TestService_MapFn_MultipleRequestsAndResponses(t *testing.T) { + svc := &Service{ + MapperStream: MapStreamerFunc(func(ctx context.Context, keys []string, datum Datum, messageCh chan<- Message) { + defer close(messageCh) + for i := 0; i < 3; i++ { // Send multiple responses for each request + msg := fmt.Sprintf("response_%d_for_%s", i, string(datum.Value())) + messageCh <- NewMessage([]byte(msg)).WithKeys([]string{keys[0] + "_test"}) } + }), + } + conn := newTestServer(t, func(server *grpc.Server) { + proto.RegisterMapServer(server, svc) + }) - }) + client := proto.NewMapClient(conn) + stream, err := client.MapFn(context.Background()) + require.NoError(t, err, "Creating stream") + + doHandshake(t, stream) + + const msgCount = 5 + for i := 0; i < msgCount; i++ { + msg := proto.MapRequest{ + Request: &proto.MapRequest_Request{ + Keys: []string{"client"}, + Value: []byte(fmt.Sprintf("test_%d", i)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + } + err = stream.Send(&msg) + require.NoError(t, err, "Sending message over the stream") + } + err = stream.CloseSend() + require.NoError(t, err, "Closing the send direction of the stream") + + expectedResults := make([][]*proto.MapResponse_Result, msgCount*3) + results := make([][]*proto.MapResponse_Result, 0) + eotCount := 0 + + for { + got, err := stream.Recv() + if err == io.EOF { + break + } + require.NoError(t, err, "Receiving message from the stream") + + if got.Status != nil && got.Status.Eot { + eotCount++ + } else { + results = append(results, got.Results) + } + } + + require.Equal(t, msgCount, eotCount, "Expected number of EOT messages") + + for i := 0; i < msgCount; i++ { + for j := 0; j < 3; j++ { + expectedResults[i*3+j] = []*proto.MapResponse_Result{ + { + Keys: []string{"client_test"}, + Value: []byte(fmt.Sprintf("response_%d_for_test_%d", j, i)), + }, + } + } } + + require.ElementsMatch(t, results, expectedResults) } diff --git a/pkg/reducer/examples/counter/go.mod b/pkg/reducer/examples/counter/go.mod index 2d704cc9..fbe62e08 100644 --- a/pkg/reducer/examples/counter/go.mod +++ b/pkg/reducer/examples/counter/go.mod @@ -1,18 +1,17 @@ module counter -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/reducer/examples/counter/go.sum b/pkg/reducer/examples/counter/go.sum index ab002170..36997a49 100644 --- a/pkg/reducer/examples/counter/go.sum +++ b/pkg/reducer/examples/counter/go.sum @@ -1,26 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/reducer/examples/sum/go.mod b/pkg/reducer/examples/sum/go.mod index f2499195..71ade673 100644 --- a/pkg/reducer/examples/sum/go.mod +++ b/pkg/reducer/examples/sum/go.mod @@ -1,18 +1,17 @@ module sum -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/reducer/examples/sum/go.sum b/pkg/reducer/examples/sum/go.sum index ab002170..36997a49 100644 --- a/pkg/reducer/examples/sum/go.sum +++ b/pkg/reducer/examples/sum/go.sum @@ -1,26 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/reducer/server.go b/pkg/reducer/server.go index e82f8039..a51ff56c 100644 --- a/pkg/reducer/server.go +++ b/pkg/reducer/server.go @@ -3,6 +3,7 @@ package reducer import ( "context" "fmt" + "log" "os/signal" "sync" "syscall" @@ -50,8 +51,10 @@ func (r *server) Start(ctx context.Context) error { defer stop() // write server info to the file + serverInfo := info.GetDefaultServerInfo() + serverInfo.MinimumNumaflowVersion = info.MinimumNumaflowVersion[info.Reducer] // start listening on unix domain socket - lis, err := shared.PrepareServer(r.opts.sockAddr, r.opts.serverInfoFilePath, info.GetDefaultServerInfo()) + lis, err := shared.PrepareServer(r.opts.sockAddr, r.opts.serverInfoFilePath, serverInfo) if err != nil { return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) } @@ -60,7 +63,6 @@ func (r *server) Start(ctx context.Context) error { // create a grpc server r.grpcServer = shared.CreateGRPCServer(r.opts.maxMessageSize) - defer r.grpcServer.GracefulStop() // register the reduce service reducepb.RegisterReduceServer(r.grpcServer, r.svc) @@ -72,6 +74,7 @@ func (r *server) Start(ctx context.Context) error { defer wg.Done() select { case <-r.shutdownCh: + log.Printf("received shutdown signal") case <-ctxWithSignal.Done(): } shared.StopGRPCServer(r.grpcServer) diff --git a/pkg/reducer/task_manager.go b/pkg/reducer/task_manager.go index d2f03730..719d26f3 100644 --- a/pkg/reducer/task_manager.go +++ b/pkg/reducer/task_manager.go @@ -3,6 +3,8 @@ package reducer import ( "context" "fmt" + "log" + "runtime/debug" "strings" v1 "github.com/numaproj/numaflow-go/pkg/apis/proto/reduce/v1" @@ -89,6 +91,7 @@ func (rtm *reduceTaskManager) CreateTask(ctx context.Context, request *v1.Reduce // handle panic defer func() { if r := recover(); r != nil { + log.Printf("panic inside reduce handler: %v %v", r, string(debug.Stack())) rtm.shutdownCh <- struct{}{} } }() diff --git a/pkg/reducestreamer/examples/counter/go.mod b/pkg/reducestreamer/examples/counter/go.mod index 2d704cc9..fbe62e08 100644 --- a/pkg/reducestreamer/examples/counter/go.mod +++ b/pkg/reducestreamer/examples/counter/go.mod @@ -1,18 +1,17 @@ module counter -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/reducestreamer/examples/counter/go.sum b/pkg/reducestreamer/examples/counter/go.sum index ab002170..36997a49 100644 --- a/pkg/reducestreamer/examples/counter/go.sum +++ b/pkg/reducestreamer/examples/counter/go.sum @@ -1,26 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/reducestreamer/examples/sum/go.mod b/pkg/reducestreamer/examples/sum/go.mod index f2499195..71ade673 100644 --- a/pkg/reducestreamer/examples/sum/go.mod +++ b/pkg/reducestreamer/examples/sum/go.mod @@ -1,18 +1,17 @@ module sum -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/reducestreamer/examples/sum/go.sum b/pkg/reducestreamer/examples/sum/go.sum index ab002170..36997a49 100644 --- a/pkg/reducestreamer/examples/sum/go.sum +++ b/pkg/reducestreamer/examples/sum/go.sum @@ -1,26 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/reducestreamer/server.go b/pkg/reducestreamer/server.go index 0d460b6f..31322933 100644 --- a/pkg/reducestreamer/server.go +++ b/pkg/reducestreamer/server.go @@ -3,6 +3,7 @@ package reducestreamer import ( "context" "fmt" + "log" "os/signal" "sync" "syscall" @@ -50,8 +51,10 @@ func (r *server) Start(ctx context.Context) error { defer stop() // write server info to the file + serverInfo := info.GetDefaultServerInfo() + serverInfo.MinimumNumaflowVersion = info.MinimumNumaflowVersion[info.Reducestreamer] // start listening on unix domain socket - lis, err := shared.PrepareServer(r.opts.sockAddr, r.opts.serverInfoFilePath, info.GetDefaultServerInfo()) + lis, err := shared.PrepareServer(r.opts.sockAddr, r.opts.serverInfoFilePath, serverInfo) if err != nil { return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) } @@ -73,6 +76,7 @@ func (r *server) Start(ctx context.Context) error { defer wg.Done() select { case <-r.shutdownCh: + log.Printf("received shutdown signal") case <-ctxWithSignal.Done(): } shared.StopGRPCServer(r.grpcServer) diff --git a/pkg/reducestreamer/task_manager.go b/pkg/reducestreamer/task_manager.go index 2a6db6e6..633d3ad3 100644 --- a/pkg/reducestreamer/task_manager.go +++ b/pkg/reducestreamer/task_manager.go @@ -3,6 +3,8 @@ package reducestreamer import ( "context" "fmt" + "log" + "runtime/debug" "strings" "sync" @@ -102,6 +104,7 @@ func (rtm *reduceStreamTaskManager) CreateTask(ctx context.Context, request *v1. // handle panic defer func() { if r := recover(); r != nil { + log.Printf("panic inside reduce handler: %v %v", r, string(debug.Stack())) rtm.shutdownCh <- struct{}{} } }() diff --git a/pkg/sessionreducer/examples/counter/go.mod b/pkg/sessionreducer/examples/counter/go.mod index 0d1177e7..29556dfe 100644 --- a/pkg/sessionreducer/examples/counter/go.mod +++ b/pkg/sessionreducer/examples/counter/go.mod @@ -1,21 +1,20 @@ module counter -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. require ( - github.com/numaproj/numaflow-go v0.8.0 + github.com/numaproj/numaflow-go v0.9.1 go.uber.org/atomic v1.11.0 ) require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sessionreducer/examples/counter/go.sum b/pkg/sessionreducer/examples/counter/go.sum index e5697aab..38feccd7 100644 --- a/pkg/sessionreducer/examples/counter/go.sum +++ b/pkg/sessionreducer/examples/counter/go.sum @@ -1,28 +1,26 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sessionreducer/examples/sum/go.mod b/pkg/sessionreducer/examples/sum/go.mod index 19fe9d5a..f57b6f8c 100644 --- a/pkg/sessionreducer/examples/sum/go.mod +++ b/pkg/sessionreducer/examples/sum/go.mod @@ -1,21 +1,20 @@ module sum -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. require ( - github.com/numaproj/numaflow-go v0.8.0 + github.com/numaproj/numaflow-go v0.9.0 go.uber.org/atomic v1.11.0 ) require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sessionreducer/examples/sum/go.sum b/pkg/sessionreducer/examples/sum/go.sum index e5697aab..38feccd7 100644 --- a/pkg/sessionreducer/examples/sum/go.sum +++ b/pkg/sessionreducer/examples/sum/go.sum @@ -1,28 +1,26 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sessionreducer/server.go b/pkg/sessionreducer/server.go index 4249e76b..f20169c3 100644 --- a/pkg/sessionreducer/server.go +++ b/pkg/sessionreducer/server.go @@ -3,6 +3,7 @@ package sessionreducer import ( "context" "fmt" + "log" "os/signal" "sync" "syscall" @@ -50,8 +51,10 @@ func (r *server) Start(ctx context.Context) error { defer stop() // write server info to the file + serverInfo := info.GetDefaultServerInfo() + serverInfo.MinimumNumaflowVersion = info.MinimumNumaflowVersion[info.Sessionreducer] // start listening on unix domain socket - lis, err := shared.PrepareServer(r.opts.sockAddr, r.opts.serverInfoFilePath, info.GetDefaultServerInfo()) + lis, err := shared.PrepareServer(r.opts.sockAddr, r.opts.serverInfoFilePath, serverInfo) if err != nil { return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) } @@ -73,6 +76,7 @@ func (r *server) Start(ctx context.Context) error { defer wg.Done() select { case <-r.shutdownCh: + log.Printf("received shutdown signal") case <-ctxWithSignal.Done(): } shared.StopGRPCServer(r.grpcServer) diff --git a/pkg/sessionreducer/task_manager.go b/pkg/sessionreducer/task_manager.go index 12e912f1..44d8d880 100644 --- a/pkg/sessionreducer/task_manager.go +++ b/pkg/sessionreducer/task_manager.go @@ -3,6 +3,8 @@ package sessionreducer import ( "context" "fmt" + "log" + "runtime/debug" "strings" "sync" @@ -134,6 +136,7 @@ func (rtm *sessionReduceTaskManager) CreateTask(ctx context.Context, request *v1 // handle panic defer func() { if r := recover(); r != nil { + log.Printf("panic inside session reduce handler: %v %v", r, string(debug.Stack())) rtm.shutdownCh <- struct{}{} } }() @@ -207,6 +210,7 @@ func (rtm *sessionReduceTaskManager) MergeTasks(ctx context.Context, request *v1 // handle panic defer func() { if r := recover(); r != nil { + log.Printf("panic inside session reduce handler: %v %v", r, string(debug.Stack())) rtm.shutdownCh <- struct{}{} } }() diff --git a/pkg/shared/util.go b/pkg/shared/util.go index 347ed1a7..1755bd8b 100644 --- a/pkg/shared/util.go +++ b/pkg/shared/util.go @@ -2,6 +2,7 @@ package shared import ( "fmt" + "log" "net" "os" "time" @@ -54,6 +55,7 @@ func StopGRPCServer(grpcServer *grpc.Server) { // if it is not stopped, stop it forcefully stopped := make(chan struct{}) go func() { + log.Printf("gracefully stopping grpc server") grpcServer.GracefulStop() close(stopped) }() @@ -61,8 +63,10 @@ func StopGRPCServer(grpcServer *grpc.Server) { t := time.NewTimer(30 * time.Second) select { case <-t.C: + log.Printf("forcefully stopping grpc server") grpcServer.Stop() case <-stopped: t.Stop() } + log.Printf("grpc server stopped") } diff --git a/pkg/sideinput/examples/map_sideinput/go.mod b/pkg/sideinput/examples/map_sideinput/go.mod index 1e085301..7a8f5b92 100644 --- a/pkg/sideinput/examples/map_sideinput/go.mod +++ b/pkg/sideinput/examples/map_sideinput/go.mod @@ -4,14 +4,13 @@ go 1.21.2 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sideinput/examples/map_sideinput/go.sum b/pkg/sideinput/examples/map_sideinput/go.sum index 917367e7..09e06a2c 100644 --- a/pkg/sideinput/examples/map_sideinput/go.sum +++ b/pkg/sideinput/examples/map_sideinput/go.sum @@ -1,29 +1,22 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/map_sideinput/udf/go.mod b/pkg/sideinput/examples/map_sideinput/udf/go.mod index 2e07e25b..f19c7c5d 100644 --- a/pkg/sideinput/examples/map_sideinput/udf/go.mod +++ b/pkg/sideinput/examples/map_sideinput/udf/go.mod @@ -6,15 +6,15 @@ replace github.com/numaproj/numaflow-go => ../../../../.. require ( github.com/fsnotify/fsnotify v1.6.0 - github.com/numaproj/numaflow-go v0.8.0 + github.com/numaproj/numaflow-go v0.9.0 ) require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sideinput/examples/map_sideinput/udf/go.sum b/pkg/sideinput/examples/map_sideinput/udf/go.sum index dbcfb187..249ac9a2 100644 --- a/pkg/sideinput/examples/map_sideinput/udf/go.sum +++ b/pkg/sideinput/examples/map_sideinput/udf/go.sum @@ -2,31 +2,26 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/reduce_sideinput/go.mod b/pkg/sideinput/examples/reduce_sideinput/go.mod index bc64ab5f..f45d0e0f 100644 --- a/pkg/sideinput/examples/reduce_sideinput/go.mod +++ b/pkg/sideinput/examples/reduce_sideinput/go.mod @@ -4,14 +4,13 @@ go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sideinput/examples/reduce_sideinput/go.sum b/pkg/sideinput/examples/reduce_sideinput/go.sum index 917367e7..09e06a2c 100644 --- a/pkg/sideinput/examples/reduce_sideinput/go.sum +++ b/pkg/sideinput/examples/reduce_sideinput/go.sum @@ -1,29 +1,22 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/reduce_sideinput/udf/go.mod b/pkg/sideinput/examples/reduce_sideinput/udf/go.mod index 877b52b8..507b2beb 100644 --- a/pkg/sideinput/examples/reduce_sideinput/udf/go.mod +++ b/pkg/sideinput/examples/reduce_sideinput/udf/go.mod @@ -6,16 +6,15 @@ replace github.com/numaproj/numaflow-go => ../../../../.. require ( github.com/fsnotify/fsnotify v1.7.0 - github.com/numaproj/numaflow-go v0.8.0 + github.com/numaproj/numaflow-go v0.9.0 ) require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sideinput/examples/reduce_sideinput/udf/go.sum b/pkg/sideinput/examples/reduce_sideinput/udf/go.sum index 18817608..917afe1a 100644 --- a/pkg/sideinput/examples/reduce_sideinput/udf/go.sum +++ b/pkg/sideinput/examples/reduce_sideinput/udf/go.sum @@ -2,32 +2,25 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/sideinput_function/go.mod b/pkg/sideinput/examples/sideinput_function/go.mod index ef2619b4..d5d4c8cd 100644 --- a/pkg/sideinput/examples/sideinput_function/go.mod +++ b/pkg/sideinput/examples/sideinput_function/go.mod @@ -4,14 +4,13 @@ go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sideinput/examples/sideinput_function/go.sum b/pkg/sideinput/examples/sideinput_function/go.sum index 917367e7..09e06a2c 100644 --- a/pkg/sideinput/examples/sideinput_function/go.sum +++ b/pkg/sideinput/examples/sideinput_function/go.sum @@ -1,29 +1,22 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/simple_sideinput/go.mod b/pkg/sideinput/examples/simple_sideinput/go.mod index ede6584d..9b330d6b 100644 --- a/pkg/sideinput/examples/simple_sideinput/go.mod +++ b/pkg/sideinput/examples/simple_sideinput/go.mod @@ -1,17 +1,16 @@ module simple_sideinput -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sideinput/examples/simple_sideinput/go.sum b/pkg/sideinput/examples/simple_sideinput/go.sum index 95c8479a..09e06a2c 100644 --- a/pkg/sideinput/examples/simple_sideinput/go.sum +++ b/pkg/sideinput/examples/simple_sideinput/go.sum @@ -1,24 +1,22 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/simple_sideinput/udf/go.mod b/pkg/sideinput/examples/simple_sideinput/udf/go.mod index 0c14f6ac..18725d1c 100644 --- a/pkg/sideinput/examples/simple_sideinput/udf/go.mod +++ b/pkg/sideinput/examples/simple_sideinput/udf/go.mod @@ -1,20 +1,20 @@ module udf -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../../.. require ( github.com/fsnotify/fsnotify v1.6.0 - github.com/numaproj/numaflow-go v0.8.0 + github.com/numaproj/numaflow-go v0.9.0 ) require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sideinput/examples/simple_sideinput/udf/go.sum b/pkg/sideinput/examples/simple_sideinput/udf/go.sum index bfde5735..249ac9a2 100644 --- a/pkg/sideinput/examples/simple_sideinput/udf/go.sum +++ b/pkg/sideinput/examples/simple_sideinput/udf/go.sum @@ -1,27 +1,27 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/simple_source_with_sideinput/go.mod b/pkg/sideinput/examples/simple_source_with_sideinput/go.mod index 700b59c1..da5107e8 100644 --- a/pkg/sideinput/examples/simple_source_with_sideinput/go.mod +++ b/pkg/sideinput/examples/simple_source_with_sideinput/go.mod @@ -6,21 +6,21 @@ replace github.com/numaproj/numaflow-go => ../../../.. require ( github.com/fsnotify/fsnotify v1.6.0 - github.com/numaproj/numaflow-go v0.8.0 - github.com/stretchr/testify v1.8.1 + github.com/numaproj/numaflow-go v0.9.0 + github.com/stretchr/testify v1.9.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/sideinput/examples/simple_source_with_sideinput/go.sum b/pkg/sideinput/examples/simple_source_with_sideinput/go.sum index f3487675..109ea570 100644 --- a/pkg/sideinput/examples/simple_source_with_sideinput/go.sum +++ b/pkg/sideinput/examples/simple_source_with_sideinput/go.sum @@ -1,15 +1,10 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -18,32 +13,25 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sideinput/examples/sink_sideinput/go.mod b/pkg/sideinput/examples/sink_sideinput/go.mod index a3b8be8a..735a05a6 100644 --- a/pkg/sideinput/examples/sink_sideinput/go.mod +++ b/pkg/sideinput/examples/sink_sideinput/go.mod @@ -7,17 +7,17 @@ replace github.com/numaproj/numaflow-go => ../../../.. require ( github.com/fsnotify/fsnotify v1.5.1 github.com/go-redis/redis/v8 v8.11.5 - github.com/numaproj/numaflow-go v0.8.0 + github.com/numaproj/numaflow-go v0.9.0 ) require ( - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sideinput/examples/sink_sideinput/go.sum b/pkg/sideinput/examples/sink_sideinput/go.sum index 80956663..26d68938 100644 --- a/pkg/sideinput/examples/sink_sideinput/go.sum +++ b/pkg/sideinput/examples/sink_sideinput/go.sum @@ -1,5 +1,5 @@ -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= @@ -8,12 +8,8 @@ github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWp github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= @@ -22,24 +18,23 @@ github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/pkg/sideinput/server.go b/pkg/sideinput/server.go index e32ab7a1..f1c0862b 100644 --- a/pkg/sideinput/server.go +++ b/pkg/sideinput/server.go @@ -3,6 +3,7 @@ package sideinput import ( "context" "fmt" + "log" "os/signal" "sync" "syscall" @@ -49,8 +50,11 @@ func (s *server) Start(ctx context.Context) error { ctxWithSignal, stop := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGTERM) defer stop() + // write server info to the file + serverInfo := info.GetDefaultServerInfo() + serverInfo.MinimumNumaflowVersion = info.MinimumNumaflowVersion[info.Sideinput] // start listening on unix domain socket - lis, err := shared.PrepareServer(s.opts.sockAddr, s.opts.serverInfoFilePath, info.GetDefaultServerInfo()) + lis, err := shared.PrepareServer(s.opts.sockAddr, s.opts.serverInfoFilePath, serverInfo) if err != nil { return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) } @@ -71,6 +75,7 @@ func (s *server) Start(ctx context.Context) error { defer wg.Done() select { case <-s.shutdownCh: + log.Printf("shutdown signal received") case <-ctxWithSignal.Done(): } shared.StopGRPCServer(s.grpcServer) diff --git a/pkg/sideinput/service.go b/pkg/sideinput/service.go index b10bdff7..7628c00e 100644 --- a/pkg/sideinput/service.go +++ b/pkg/sideinput/service.go @@ -2,6 +2,8 @@ package sideinput import ( "context" + "log" + "runtime/debug" "google.golang.org/protobuf/types/known/emptypb" @@ -33,6 +35,7 @@ func (fs *Service) RetrieveSideInput(ctx context.Context, _ *emptypb.Empty) (*si // handle panic defer func() { if r := recover(); r != nil { + log.Printf("panic inside sideinput handler: %v %v", r, string(debug.Stack())) fs.shutdownCh <- struct{}{} } }() diff --git a/pkg/sinker/examples/fallback/go.mod b/pkg/sinker/examples/fallback/go.mod index 893f1ee3..1cd7e07b 100644 --- a/pkg/sinker/examples/fallback/go.mod +++ b/pkg/sinker/examples/fallback/go.mod @@ -1,17 +1,17 @@ module fallback -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sinker/examples/fallback/go.sum b/pkg/sinker/examples/fallback/go.sum index 95c8479a..36997a49 100644 --- a/pkg/sinker/examples/fallback/go.sum +++ b/pkg/sinker/examples/fallback/go.sum @@ -1,24 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sinker/examples/log/go.mod b/pkg/sinker/examples/log/go.mod index b571d4d3..ac86c784 100644 --- a/pkg/sinker/examples/log/go.mod +++ b/pkg/sinker/examples/log/go.mod @@ -1,17 +1,17 @@ module log_sink -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sinker/examples/log/go.sum b/pkg/sinker/examples/log/go.sum index 95c8479a..36997a49 100644 --- a/pkg/sinker/examples/log/go.sum +++ b/pkg/sinker/examples/log/go.sum @@ -1,24 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sinker/examples/redis_sink/go.mod b/pkg/sinker/examples/redis_sink/go.mod index 3e62dceb..98192b69 100644 --- a/pkg/sinker/examples/redis_sink/go.mod +++ b/pkg/sinker/examples/redis_sink/go.mod @@ -1,23 +1,23 @@ module redis-e2e-test-sink -go 1.18 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. require ( github.com/go-redis/redis/v8 v8.11.5 - github.com/numaproj/numaflow-go v0.8.0 + github.com/numaproj/numaflow-go v0.9.0 ) require ( - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sinker/examples/redis_sink/go.sum b/pkg/sinker/examples/redis_sink/go.sum index ad93d13a..26d68938 100644 --- a/pkg/sinker/examples/redis_sink/go.sum +++ b/pkg/sinker/examples/redis_sink/go.sum @@ -1,38 +1,43 @@ -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sinker/server.go b/pkg/sinker/server.go index 3e298f66..82987bb3 100644 --- a/pkg/sinker/server.go +++ b/pkg/sinker/server.go @@ -3,6 +3,7 @@ package sinker import ( "context" "fmt" + "log" "os/signal" "sync" "syscall" @@ -48,8 +49,10 @@ func NewServer(h Sinker, inputOptions ...Option) numaflow.Server { func (s *sinkServer) Start(ctx context.Context) error { // write server info to the file + serverInfo := info.GetDefaultServerInfo() + serverInfo.MinimumNumaflowVersion = info.MinimumNumaflowVersion[info.Sinker] // start listening on unix domain socket - lis, err := shared.PrepareServer(s.opts.sockAddr, s.opts.serverInfoFilePath, info.GetDefaultServerInfo()) + lis, err := shared.PrepareServer(s.opts.sockAddr, s.opts.serverInfoFilePath, serverInfo) if err != nil { return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) } @@ -74,6 +77,7 @@ func (s *sinkServer) Start(ctx context.Context) error { defer wg.Done() select { case <-s.shutdownCh: + log.Printf("shutdown signal received") case <-ctxWithSignal.Done(): } shared.StopGRPCServer(s.grpcServer) diff --git a/pkg/sinker/service.go b/pkg/sinker/service.go index d775cbdd..8c8c7f9a 100644 --- a/pkg/sinker/service.go +++ b/pkg/sinker/service.go @@ -2,10 +2,14 @@ package sinker import ( "context" + "errors" + "fmt" "io" - "sync" + "log" + "runtime/debug" "time" + "golang.org/x/sync/errgroup" "google.golang.org/protobuf/types/known/emptypb" sinkpb "github.com/numaproj/numaflow-go/pkg/apis/proto/sink/v1" @@ -70,68 +74,169 @@ func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*sinkpb.ReadyRespon // SinkFn applies a sink function to a every element. func (fs *Service) SinkFn(stream sinkpb.Sink_SinkFnServer) error { - var ( - resultList []*sinkpb.SinkResponse_Result - wg sync.WaitGroup - datumStreamCh = make(chan Datum) - ctx = stream.Context() - ) - - wg.Add(1) - go func() { - defer wg.Done() - // handle panic - defer func() { - if r := recover(); r != nil { - fs.shutdownCh <- struct{}{} - } - }() - messages := fs.Sinker.Sink(ctx, datumStreamCh) - for _, msg := range messages { - if msg.Fallback { - resultList = append(resultList, &sinkpb.SinkResponse_Result{ - Id: msg.ID, - Status: sinkpb.Status_FALLBACK, - }) - } else if msg.Success { - resultList = append(resultList, &sinkpb.SinkResponse_Result{ - Id: msg.ID, - Status: sinkpb.Status_SUCCESS, - }) - } else { - resultList = append(resultList, &sinkpb.SinkResponse_Result{ - Id: msg.ID, - Status: sinkpb.Status_FAILURE, - ErrMsg: msg.Err, - }) + ctx := stream.Context() + + // Perform handshake before entering the main loop + if err := fs.performHandshake(stream); err != nil { + return err + } + + for { + datumStreamCh := make(chan Datum) + g, groupCtx := errgroup.WithContext(ctx) + + g.Go(func() error { + return fs.receiveRequests(groupCtx, stream, datumStreamCh) + }) + + g.Go(func() error { + return fs.processData(groupCtx, stream, datumStreamCh) + }) + + // Wait for the goroutines to finish + if err := g.Wait(); err != nil { + if errors.Is(err, io.EOF) { + log.Printf("Stopping the SinkFn") + return nil } + log.Printf("Stopping the SinkFn with err, %s", err) + fs.shutdownCh <- struct{}{} + return err } + } +} + +// performHandshake performs the handshake with the client. +func (fs *Service) performHandshake(stream sinkpb.Sink_SinkFnServer) error { + req, err := stream.Recv() + if err != nil { + log.Printf("error receiving handshake from stream: %v", err) + return err + } + + if req.Handshake == nil || !req.Handshake.Sot { + return fmt.Errorf("expected handshake message") + } + + handshakeResponse := &sinkpb.SinkResponse{ + Handshake: &sinkpb.Handshake{ + Sot: true, + }, + } + if err := stream.Send(handshakeResponse); err != nil { + return err + } + + return nil +} + +// recvWithContext wraps stream.Recv() to respect context cancellation. +func recvWithContext(ctx context.Context, stream sinkpb.Sink_SinkFnServer) (*sinkpb.SinkRequest, error) { + type recvResult struct { + req *sinkpb.SinkRequest + err error + } + + resultCh := make(chan recvResult, 1) + go func() { + req, err := stream.Recv() + resultCh <- recvResult{req: req, err: err} }() + select { + case <-ctx.Done(): + return nil, ctx.Err() + case result := <-resultCh: + return result.req, result.err + } +} + +// receiveRequests receives the requests from the client writes them to the datumStreamCh channel. +func (fs *Service) receiveRequests(ctx context.Context, stream sinkpb.Sink_SinkFnServer, datumStreamCh chan<- Datum) error { + defer close(datumStreamCh) + for { - d, err := stream.Recv() + req, err := recvWithContext(ctx, stream) if err == io.EOF { - close(datumStreamCh) - break + log.Printf("end of sink stream") + return err } if err != nil { - close(datumStreamCh) - // TODO: research on gRPC errors and revisit the error handler + log.Printf("error receiving from sink stream: %v", err) return err } - var hd = &handlerDatum{ - id: d.GetId(), - value: d.GetValue(), - keys: d.GetKeys(), - eventTime: d.GetEventTime().AsTime(), - watermark: d.GetWatermark().AsTime(), - headers: d.GetHeaders(), + + if req.Status != nil && req.Status.Eot { + break + } + + datum := &handlerDatum{ + id: req.GetRequest().GetId(), + value: req.GetRequest().GetValue(), + keys: req.GetRequest().GetKeys(), + eventTime: req.GetRequest().GetEventTime().AsTime(), + watermark: req.GetRequest().GetWatermark().AsTime(), + headers: req.GetRequest().GetHeaders(), + } + + select { + case <-ctx.Done(): + return nil + case datumStreamCh <- datum: } - datumStreamCh <- hd } + return nil +} - wg.Wait() - return stream.SendAndClose(&sinkpb.SinkResponse{ +// processData invokes the sinker to process the data and sends the response back to the client. +func (fs *Service) processData(ctx context.Context, stream sinkpb.Sink_SinkFnServer, datumStreamCh chan Datum) (err error) { + defer func() { + if r := recover(); r != nil { + log.Printf("panic inside sink handler: %v %v", r, string(debug.Stack())) + err = fmt.Errorf("panic inside sink handler: %v", r) + } + }() + responses := fs.Sinker.Sink(ctx, datumStreamCh) + + var resultList []*sinkpb.SinkResponse_Result + for _, msg := range responses { + if msg.Fallback { + resultList = append(resultList, &sinkpb.SinkResponse_Result{ + Id: msg.ID, + Status: sinkpb.Status_FALLBACK, + }) + } else if msg.Success { + resultList = append(resultList, &sinkpb.SinkResponse_Result{ + Id: msg.ID, + Status: sinkpb.Status_SUCCESS, + }) + } else { + resultList = append(resultList, &sinkpb.SinkResponse_Result{ + Id: msg.ID, + Status: sinkpb.Status_FAILURE, + ErrMsg: msg.Err, + }) + } + } + if err := stream.Send(&sinkpb.SinkResponse{ Results: resultList, - }) + }); err != nil { + log.Printf("error sending sink response: %v", err) + return err + } + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + // send the end of transmission message + eotResponse := &sinkpb.SinkResponse{ + Status: &sinkpb.TransmissionStatus{Eot: true}, + } + if err := stream.Send(eotResponse); err != nil { + log.Printf("error sending end of transmission message: %v", err) + return err + } + return nil } diff --git a/pkg/sinker/service_test.go b/pkg/sinker/service_test.go index d6ad1b5b..25917dc1 100644 --- a/pkg/sinker/service_test.go +++ b/pkg/sinker/service_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "google.golang.org/grpc" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" @@ -19,12 +20,12 @@ import ( type SinkFnServerTest struct { ctx context.Context inputCh chan *sinkpb.SinkRequest - rl *sinkpb.SinkResponse + rl []*sinkpb.SinkResponse grpc.ServerStream } -func (t *SinkFnServerTest) SendAndClose(list *sinkpb.SinkResponse) error { - t.rl = list +func (t *SinkFnServerTest) Send(response *sinkpb.SinkResponse) error { + t.rl = append(t.rl, response) return nil } @@ -45,35 +46,49 @@ func TestService_SinkFn(t *testing.T) { name string sh Sinker input []*sinkpb.SinkRequest - expected []*sinkpb.SinkResponse_Result + expected []*sinkpb.SinkResponse }{ { name: "sink_fn_test_success", input: []*sinkpb.SinkRequest{ { - Id: "one-processed", - Keys: []string{"sink-test"}, - Value: []byte(strconv.Itoa(10)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - Headers: map[string]string{"x-txn-id": "test-txn-1"}, + Handshake: &sinkpb.Handshake{ + Sot: true, + }, }, { - Id: "two-processed", - Keys: []string{"sink-test"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - Headers: map[string]string{"x-txn-id": "test-txn-2"}, + Request: &sinkpb.SinkRequest_Request{ + Id: "one-processed", + Keys: []string{"sink-test"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + Headers: map[string]string{"x-txn-id": "test-txn-1"}, + }, }, { - Id: "three-processed", - Keys: []string{"sink-test"}, - Value: []byte(strconv.Itoa(30)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - Headers: map[string]string{"x-txn-id": "test-txn-3"}, + Request: &sinkpb.SinkRequest_Request{ + Id: "two-processed", + Keys: []string{"sink-test"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + Headers: map[string]string{"x-txn-id": "test-txn-2"}, + }, + }, + { + Request: &sinkpb.SinkRequest_Request{ + Id: "three-processed", + Keys: []string{"sink-test"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + Headers: map[string]string{"x-txn-id": "test-txn-3"}, + }, + }, + { + Status: &sinkpb.TransmissionStatus{Eot: true}, }, }, sh: SinkerFunc(func(ctx context.Context, rch <-chan Datum) Responses { @@ -84,50 +99,75 @@ func TestService_SinkFn(t *testing.T) { } return result }), - expected: []*sinkpb.SinkResponse_Result{ + expected: []*sinkpb.SinkResponse{ { - Status: sinkpb.Status_SUCCESS, - Id: "one-processed", - ErrMsg: "", + Handshake: &sinkpb.Handshake{ + Sot: true, + }, }, { - Status: sinkpb.Status_SUCCESS, - Id: "two-processed", - ErrMsg: "", + Results: []*sinkpb.SinkResponse_Result{ + { + Status: sinkpb.Status_SUCCESS, + Id: "one-processed", + ErrMsg: "", + }, + { + Status: sinkpb.Status_SUCCESS, + Id: "two-processed", + ErrMsg: "", + }, + { + Status: sinkpb.Status_SUCCESS, + Id: "three-processed", + ErrMsg: "", + }, + }, }, { - Status: sinkpb.Status_SUCCESS, - Id: "three-processed", - ErrMsg: "", + Status: &sinkpb.TransmissionStatus{Eot: true}, }, }, }, { name: "sink_fn_test_failure", - input: []*sinkpb.SinkRequest{ { - Id: "one-processed", - Keys: []string{"sink-test-1", "sink-test-2"}, - Value: []byte(strconv.Itoa(10)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - Headers: map[string]string{"x-txn-id": "test-txn-1"}, + Handshake: &sinkpb.Handshake{ + Sot: true, + }, + }, + { + Request: &sinkpb.SinkRequest_Request{ + Id: "one-processed", + Keys: []string{"sink-test-1", "sink-test-2"}, + Value: []byte(strconv.Itoa(10)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + Headers: map[string]string{"x-txn-id": "test-txn-1"}, + }, + }, + { + Request: &sinkpb.SinkRequest_Request{ + Id: "two-processed", + Keys: []string{"sink-test-1", "sink-test-2"}, + Value: []byte(strconv.Itoa(20)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, }, { - Id: "two-processed", - Keys: []string{"sink-test-1", "sink-test-2"}, - Value: []byte(strconv.Itoa(20)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + Request: &sinkpb.SinkRequest_Request{ + Id: "three-processed", + Keys: []string{"sink-test-1", "sink-test-2"}, + Value: []byte(strconv.Itoa(30)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + Headers: map[string]string{"x-txn-id": "test-txn-2"}, + }, }, { - Id: "three-processed", - Keys: []string{"sink-test-1", "sink-test-2"}, - Value: []byte(strconv.Itoa(30)), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), - Headers: map[string]string{"x-txn-id": "test-txn-2"}, + Status: &sinkpb.TransmissionStatus{Eot: true}, }, }, sh: SinkerFunc(func(ctx context.Context, rch <-chan Datum) Responses { @@ -138,21 +178,33 @@ func TestService_SinkFn(t *testing.T) { } return result }), - expected: []*sinkpb.SinkResponse_Result{ + expected: []*sinkpb.SinkResponse{ { - Status: sinkpb.Status_FAILURE, - Id: "one-processed", - ErrMsg: "unknown error", + Handshake: &sinkpb.Handshake{ + Sot: true, + }, }, { - Status: sinkpb.Status_FAILURE, - Id: "two-processed", - ErrMsg: "unknown error", + Results: []*sinkpb.SinkResponse_Result{ + { + Status: sinkpb.Status_FAILURE, + Id: "one-processed", + ErrMsg: "unknown error", + }, + { + Status: sinkpb.Status_FAILURE, + Id: "two-processed", + ErrMsg: "unknown error", + }, + { + Status: sinkpb.Status_FAILURE, + Id: "three-processed", + ErrMsg: "unknown error", + }, + }, }, { - Status: sinkpb.Status_FAILURE, - Id: "three-processed", - ErrMsg: "unknown error", + Status: &sinkpb.TransmissionStatus{Eot: true}, }, }, }, @@ -173,7 +225,8 @@ func TestService_SinkFn(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - _ = ss.SinkFn(udfReduceFnStream) + err := ss.SinkFn(udfReduceFnStream) + assert.NoError(t, err) }() for _, val := range tt.input { @@ -183,8 +236,8 @@ func TestService_SinkFn(t *testing.T) { wg.Wait() - if !reflect.DeepEqual(udfReduceFnStream.rl.Results, tt.expected) { - t.Errorf("ReduceFn() got = %v, want %v", udfReduceFnStream.rl.Results, tt.expected) + for i, val := range tt.expected { + assert.Equal(t, val, udfReduceFnStream.rl[i]) } }) } diff --git a/pkg/sourcer/examples/simple_source/go.mod b/pkg/sourcer/examples/simple_source/go.mod index 56e8dc97..2a8b4f7f 100644 --- a/pkg/sourcer/examples/simple_source/go.mod +++ b/pkg/sourcer/examples/simple_source/go.mod @@ -1,26 +1,26 @@ module simple_source -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. require ( - github.com/google/uuid v1.3.0 - github.com/numaproj/numaflow-go v0.8.0 - github.com/stretchr/testify v1.8.1 + github.com/google/uuid v1.6.0 + github.com/numaproj/numaflow-go v0.9.0 + github.com/stretchr/testify v1.9.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.12.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/sourcer/examples/simple_source/go.sum b/pkg/sourcer/examples/simple_source/go.sum index 7e7e9a2c..b9129942 100644 --- a/pkg/sourcer/examples/simple_source/go.sum +++ b/pkg/sourcer/examples/simple_source/go.sum @@ -1,45 +1,36 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sourcer/examples/simple_source/impl/simple_source.go b/pkg/sourcer/examples/simple_source/impl/simple_source.go index b294850d..33423ccf 100644 --- a/pkg/sourcer/examples/simple_source/impl/simple_source.go +++ b/pkg/sourcer/examples/simple_source/impl/simple_source.go @@ -27,8 +27,10 @@ func NewSimpleSource() *SimpleSource { } func (s *SimpleSource) Pending(_ context.Context) int64 { + s.lock.Lock() + defer s.lock.Unlock() // The simple source always returns zero to indicate there is no pending record. - return 0 + return int64(len(s.toAckSet)) } func (s *SimpleSource) Read(_ context.Context, readRequest sourcesdk.ReadRequest, messageCh chan<- sourcesdk.Message) { @@ -72,6 +74,8 @@ func (s *SimpleSource) Read(_ context.Context, readRequest sourcesdk.ReadRequest } func (s *SimpleSource) Ack(_ context.Context, request sourcesdk.AckRequest) { + s.lock.Lock() + defer s.lock.Unlock() for _, offset := range request.Offsets() { delete(s.toAckSet, deserializeOffset(offset.Value())) } diff --git a/pkg/sourcer/interface.go b/pkg/sourcer/interface.go index 01f78515..3c7845a3 100644 --- a/pkg/sourcer/interface.go +++ b/pkg/sourcer/interface.go @@ -36,6 +36,6 @@ type ReadRequest interface { // AckRequest is the interface of ack request. type AckRequest interface { - // Offsets returns the offsets of the records to ack. + // Offsets returns the offsets to be acknowledged. Offsets() []Offset } diff --git a/pkg/sourcer/server.go b/pkg/sourcer/server.go index 247fa37a..0f759a79 100644 --- a/pkg/sourcer/server.go +++ b/pkg/sourcer/server.go @@ -3,6 +3,7 @@ package sourcer import ( "context" "fmt" + "log" "os/signal" "sync" "syscall" @@ -48,10 +49,11 @@ func NewServer( // Start starts the gRPC server via unix domain socket at shared.address and return error. func (s *server) Start(ctx context.Context) error { - // write server info to the file + serverInfo := info.GetDefaultServerInfo() + serverInfo.MinimumNumaflowVersion = info.MinimumNumaflowVersion[info.Sourcer] // start listening on unix domain socket - lis, err := shared.PrepareServer(s.opts.sockAddr, s.opts.serverInfoFilePath, info.GetDefaultServerInfo()) + lis, err := shared.PrepareServer(s.opts.sockAddr, s.opts.serverInfoFilePath, serverInfo) if err != nil { return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) } @@ -76,6 +78,7 @@ func (s *server) Start(ctx context.Context) error { defer wg.Done() select { case <-s.shutdownCh: + log.Printf("shutdown signal received") case <-ctxWithSignal.Done(): } shared.StopGRPCServer(s.grpcServer) diff --git a/pkg/sourcer/service.go b/pkg/sourcer/service.go index e4e5426d..9453c014 100644 --- a/pkg/sourcer/service.go +++ b/pkg/sourcer/service.go @@ -2,8 +2,14 @@ package sourcer import ( "context" + "errors" + "fmt" + "io" + "log" + "runtime/debug" "time" + "golang.org/x/sync/errgroup" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" @@ -24,81 +30,162 @@ type Service struct { shutdownCh chan<- struct{} } -// IsReady returns true to indicate the gRPC connection is ready. -func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*sourcepb.ReadyResponse, error) { - return &sourcepb.ReadyResponse{Ready: true}, nil -} +// ReadFn reads the data from the source. +func (fs *Service) ReadFn(stream sourcepb.Source_ReadFnServer) error { + ctx := stream.Context() -// PendingFn returns the number of pending messages. -func (fs *Service) PendingFn(ctx context.Context, _ *emptypb.Empty) (*sourcepb.PendingResponse, error) { - // handle panic - defer func() { - if r := recover(); r != nil { + if err := fs.performReadHandshake(stream); err != nil { + return err + } + + for { + if err := fs.receiveReadRequests(ctx, stream); err != nil { + // If the error is EOF, it means the stream has been closed. + if errors.Is(err, io.EOF) { + return nil + } + log.Printf("error processing requests: %v", err) fs.shutdownCh <- struct{}{} + return err } - }() - - return &sourcepb.PendingResponse{Result: &sourcepb.PendingResponse_Result{ - Count: fs.Source.Pending(ctx), - }}, nil + } } -// readRequest implements the ReadRequest interface and is used in the read handler. -type readRequest struct { - count uint64 - timeout time.Duration -} +// performReadHandshake performs the handshake with the client before starting the read process. +func (fs *Service) performReadHandshake(stream sourcepb.Source_ReadFnServer) error { + req, err := stream.Recv() + if err != nil { + log.Printf("error receiving handshake from stream: %v", err) + return err + } -func (r *readRequest) TimeOut() time.Duration { - return r.timeout -} + if req.Handshake == nil || !req.Handshake.Sot { + return fmt.Errorf("expected handshake message") + } -func (r *readRequest) Count() uint64 { - return r.count + handshakeResponse := &sourcepb.ReadResponse{ + Status: &sourcepb.ReadResponse_Status{ + Eot: false, + Code: sourcepb.ReadResponse_Status_SUCCESS, + }, + Handshake: &sourcepb.Handshake{ + Sot: true, + }, + } + if err := stream.Send(handshakeResponse); err != nil { + return err + } + + return nil } -// ReadFn reads the data from the source. -func (fs *Service) ReadFn(d *sourcepb.ReadRequest, stream sourcepb.Source_ReadFnServer) error { - request := readRequest{ - count: d.Request.GetNumRecords(), - timeout: time.Duration(d.Request.GetTimeoutInMs()) * time.Millisecond, +// recvWithContext wraps stream.Recv() to respect context cancellation for ReadFn. +func recvWithContextRead(ctx context.Context, stream sourcepb.Source_ReadFnServer) (*sourcepb.ReadRequest, error) { + type recvResult struct { + req *sourcepb.ReadRequest + err error } - ctx := stream.Context() - messageCh := make(chan Message) - // Start the read in a goroutine + resultCh := make(chan recvResult, 1) go func() { - defer close(messageCh) + req, err := stream.Recv() + resultCh <- recvResult{req: req, err: err} + }() + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case result := <-resultCh: + return result.req, result.err + } +} + +// receiveReadRequests receives read requests from the client and invokes the source Read method. +// writes the read data to the message channel. +func (fs *Service) receiveReadRequests(ctx context.Context, stream sourcepb.Source_ReadFnServer) error { + messageCh := make(chan Message) + eg, groupCtx := errgroup.WithContext(ctx) + req, err := recvWithContextRead(groupCtx, stream) + if err == io.EOF { + log.Printf("end of read stream") + return err + } + if err != nil { + log.Printf("error receiving from read stream: %v", err) + return err + } + + eg.Go(func() (err error) { // handle panic defer func() { if r := recover(); r != nil { - fs.shutdownCh <- struct{}{} + log.Printf("panic inside source handler: %v %v", r, string(debug.Stack())) + err = fmt.Errorf("panic inside source handler: %v", r) + return } + close(messageCh) }() + request := readRequest{ + count: req.Request.GetNumRecords(), + timeout: time.Duration(req.Request.GetTimeoutInMs()) * time.Millisecond, + } fs.Source.Read(ctx, &request, messageCh) - }() + return nil + }) - // Read messages from the channel and send them to the stream, until the channel is closed - for msg := range messageCh { - offset := &sourcepb.Offset{ - Offset: msg.Offset().Value(), - PartitionId: msg.Offset().PartitionId(), - } - element := &sourcepb.ReadResponse{ - Result: &sourcepb.ReadResponse_Result{ - Payload: msg.Value(), - Offset: offset, - EventTime: timestamppb.New(msg.EventTime()), - Keys: msg.Keys(), - Headers: msg.Headers(), - }, - } - // The error here is returned by the stream, which is already a gRPC error - if err := stream.Send(element); err != nil { - // The channel may or may not be closed, as we are not sure, we leave it to GC. - return err + // invoke the processReadData method to send the read data to the client. + eg.Go(func() error { + return fs.processReadData(groupCtx, stream, messageCh) + }) + + if err := eg.Wait(); err != nil { + return err + } + return nil +} + +// processReadData processes the read data and sends it to the client. +func (fs *Service) processReadData(ctx context.Context, stream sourcepb.Source_ReadFnServer, messageCh <-chan Message) error { +readLoop: + for { + select { + case <-ctx.Done(): + return ctx.Err() + case msg, ok := <-messageCh: + if !ok { + break readLoop + } + offset := &sourcepb.Offset{ + Offset: msg.Offset().Value(), + PartitionId: msg.Offset().PartitionId(), + } + element := &sourcepb.ReadResponse{ + Result: &sourcepb.ReadResponse_Result{ + Payload: msg.Value(), + Offset: offset, + EventTime: timestamppb.New(msg.EventTime()), + Keys: msg.Keys(), + Headers: msg.Headers(), + }, + Status: &sourcepb.ReadResponse_Status{ + Eot: false, + Code: 0, + }, + } + if err := stream.Send(element); err != nil { + return err + } } } + err := stream.Send(&sourcepb.ReadResponse{ + Status: &sourcepb.ReadResponse_Status{ + Eot: true, + Code: 0, + }, + }) + if err != nil { + return err + } return nil } @@ -107,22 +194,101 @@ type ackRequest struct { offsets []Offset } -// Offsets returns the offsets of the records to ack. +// Offsets returns the offsets to be acknowledged. func (a *ackRequest) Offsets() []Offset { return a.offsets } -// AckFn applies a function to each datum element. -func (fs *Service) AckFn(ctx context.Context, d *sourcepb.AckRequest) (*sourcepb.AckResponse, error) { - // handle panic +// AckFn acknowledges the data from the source. +func (fs *Service) AckFn(stream sourcepb.Source_AckFnServer) error { + ctx := stream.Context() + + if err := fs.performAckHandshake(stream); err != nil { + return err + } + + for { + err := fs.receiveAckRequests(ctx, stream) + + if err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + } +} + +// performAckHandshake performs the handshake with the client before starting the ack process. +func (fs *Service) performAckHandshake(stream sourcepb.Source_AckFnServer) error { + req, err := stream.Recv() + if err != nil { + log.Printf("error receiving handshake from stream: %v", err) + return err + } + + if req.Handshake == nil || !req.Handshake.Sot { + return fmt.Errorf("expected handshake message") + } + + handshakeResponse := &sourcepb.AckResponse{ + Result: &sourcepb.AckResponse_Result{ + Success: &emptypb.Empty{}, + }, + Handshake: &sourcepb.Handshake{ + Sot: true, + }, + } + if err := stream.Send(handshakeResponse); err != nil { + return err + } + + return nil +} + +// recvWithContext wraps stream.Recv() to respect context cancellation for AckFn. +func recvWithContextAck(ctx context.Context, stream sourcepb.Source_AckFnServer) (*sourcepb.AckRequest, error) { + type recvResult struct { + req *sourcepb.AckRequest + err error + } + + resultCh := make(chan recvResult, 1) + go func() { + req, err := stream.Recv() + resultCh <- recvResult{req: req, err: err} + }() + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case result := <-resultCh: + return result.req, result.err + } +} + +// receiveAckRequests receives ack requests from the client and invokes the source Ack method. +func (fs *Service) receiveAckRequests(ctx context.Context, stream sourcepb.Source_AckFnServer) (err error) { defer func() { if r := recover(); r != nil { + log.Printf("panic inside source handler: %v %v", r, string(debug.Stack())) fs.shutdownCh <- struct{}{} + err = fmt.Errorf("panic inside source handler: %v", r) } }() - offsets := make([]Offset, len(d.Request.GetOffsets())) - for i, offset := range d.Request.GetOffsets() { + req, err := recvWithContextAck(ctx, stream) + if err == io.EOF { + log.Printf("end of ack stream") + return err + } + if err != nil { + log.Printf("error receiving from ack stream: %v", err) + return err + } + + offsets := make([]Offset, len(req.Request.GetOffsets())) + for i, offset := range req.Request.GetOffsets() { offsets[i] = NewOffset(offset.GetOffset(), offset.GetPartitionId()) } @@ -130,15 +296,58 @@ func (fs *Service) AckFn(ctx context.Context, d *sourcepb.AckRequest) (*sourcepb offsets: offsets, } fs.Source.Ack(ctx, &request) - return &sourcepb.AckResponse{ - Result: &sourcepb.AckResponse_Result{}, - }, nil + + ackResponse := &sourcepb.AckResponse{ + Result: &sourcepb.AckResponse_Result{ + Success: &emptypb.Empty{}, + }, + } + if err := stream.Send(ackResponse); err != nil { + log.Printf("error sending ack response: %v", err) + return err + } + return nil +} + +// IsReady returns true to indicate the gRPC connection is ready. +func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*sourcepb.ReadyResponse, error) { + return &sourcepb.ReadyResponse{Ready: true}, nil +} + +// PendingFn returns the number of pending messages. +func (fs *Service) PendingFn(ctx context.Context, _ *emptypb.Empty) (*sourcepb.PendingResponse, error) { + // handle panic + defer func() { + if r := recover(); r != nil { + log.Printf("panic inside sourcer handler: %v %v", r, string(debug.Stack())) + fs.shutdownCh <- struct{}{} + } + }() + + return &sourcepb.PendingResponse{Result: &sourcepb.PendingResponse_Result{ + Count: fs.Source.Pending(ctx), + }}, nil +} + +// readRequest implements the ReadRequest interface and is used in the read handler. +type readRequest struct { + count uint64 + timeout time.Duration +} + +func (r *readRequest) TimeOut() time.Duration { + return r.timeout +} + +func (r *readRequest) Count() uint64 { + return r.count } func (fs *Service) PartitionsFn(ctx context.Context, _ *emptypb.Empty) (*sourcepb.PartitionsResponse, error) { // handle panic defer func() { if r := recover(); r != nil { + log.Printf("panic inside source handler: %v %v", r, string(debug.Stack())) fs.shutdownCh <- struct{}{} } }() diff --git a/pkg/sourcer/service_test.go b/pkg/sourcer/service_test.go index fe6643c8..bf3ea4bc 100644 --- a/pkg/sourcer/service_test.go +++ b/pkg/sourcer/service_test.go @@ -3,6 +3,7 @@ package sourcer import ( "context" "fmt" + "io" "reflect" "sync" "testing" @@ -54,22 +55,35 @@ func TestService_IsReady(t *testing.T) { type ReadFnServerTest struct { ctx context.Context - outputCh chan sourcepb.ReadResponse + outputCh chan *sourcepb.ReadResponse + requests []*sourcepb.ReadRequest grpc.ServerStream + index int +} + +func (t *ReadFnServerTest) Recv() (*sourcepb.ReadRequest, error) { + if t.index >= len(t.requests) { + return nil, io.EOF + } + req := t.requests[t.index] + t.index++ + return req, nil } func NewReadFnServerTest( ctx context.Context, - outputCh chan sourcepb.ReadResponse, + outputCh chan *sourcepb.ReadResponse, + requests []*sourcepb.ReadRequest, ) *ReadFnServerTest { return &ReadFnServerTest{ ctx: ctx, outputCh: outputCh, + requests: requests, } } func (t *ReadFnServerTest) Send(d *sourcepb.ReadResponse) error { - t.outputCh <- *d + t.outputCh <- d return nil } @@ -82,6 +96,10 @@ type ReadFnServerErrTest struct { grpc.ServerStream } +func (te *ReadFnServerErrTest) Recv() (*sourcepb.ReadRequest, error) { + return nil, fmt.Errorf("recv error") +} + func NewReadFnServerErrTest( ctx context.Context, ) *ReadFnServerErrTest { @@ -98,43 +116,74 @@ func (te *ReadFnServerErrTest) Context() context.Context { return te.ctx } +type AckFnServerTest struct { + ctx context.Context + offsets []*sourcepb.Offset + responses []*sourcepb.AckResponse + grpc.ServerStream + index int + handshakeDone bool +} + +func (a *AckFnServerTest) Recv() (*sourcepb.AckRequest, error) { + if !a.handshakeDone { + a.handshakeDone = true + return &sourcepb.AckRequest{ + Handshake: &sourcepb.Handshake{ + Sot: true, + }, + }, nil + } + if a.index >= len(a.offsets) { + return nil, io.EOF + } + offset := a.offsets[a.index] + a.index++ + return &sourcepb.AckRequest{ + Request: &sourcepb.AckRequest_Request{ + Offsets: []*sourcepb.Offset{offset}, + }, + }, nil +} + +func (a *AckFnServerTest) Send(response *sourcepb.AckResponse) error { + a.responses = append(a.responses, response) + return nil +} + +func NewAckFnServerTest( + ctx context.Context, + offsets []*sourcepb.Offset, +) *AckFnServerTest { + return &AckFnServerTest{ + ctx: ctx, + offsets: offsets, + responses: make([]*sourcepb.AckResponse, 0), + } +} + +func (a *AckFnServerTest) Context() context.Context { + return a.ctx +} + func TestService_ReadFn(t *testing.T) { tests := []struct { name string - input *sourcepb.ReadRequest expected []*sourcepb.ReadResponse expectedErr bool }{ { name: "read_fn_read_msg", - input: &sourcepb.ReadRequest{ - Request: &sourcepb.ReadRequest_Request{ - NumRecords: 1, - TimeoutInMs: 1000, - }, - }, expected: []*sourcepb.ReadResponse{ { - Result: &sourcepb.ReadResponse_Result{ - Payload: []byte(`test`), - Offset: &sourcepb.Offset{}, - EventTime: timestamppb.New(testEventTime), - Keys: []string{testKey}, - Headers: map[string]string{"x-txn-id": "test-txn-id"}, + Status: &sourcepb.ReadResponse_Status{ + Eot: false, + Code: sourcepb.ReadResponse_Status_SUCCESS, + }, + Handshake: &sourcepb.Handshake{ + Sot: true, }, }, - }, - expectedErr: false, - }, - { - name: "read_fn_err", - input: &sourcepb.ReadRequest{ - Request: &sourcepb.ReadRequest_Request{ - NumRecords: 1, - TimeoutInMs: 1000, - }, - }, - expected: []*sourcepb.ReadResponse{ { Result: &sourcepb.ReadResponse_Result{ Payload: []byte(`test`), @@ -143,8 +192,23 @@ func TestService_ReadFn(t *testing.T) { Keys: []string{testKey}, Headers: map[string]string{"x-txn-id": "test-txn-id"}, }, + Status: &sourcepb.ReadResponse_Status{ + Eot: false, + Code: sourcepb.ReadResponse_Status_SUCCESS, + }, + }, + { + Status: &sourcepb.ReadResponse_Status{ + Eot: true, + Code: sourcepb.ReadResponse_Status_SUCCESS, + }, }, }, + expectedErr: false, + }, + { + name: "read_fn_err", + expected: []*sourcepb.ReadResponse{}, expectedErr: true, }, } @@ -155,14 +219,25 @@ func TestService_ReadFn(t *testing.T) { // because we are not using gRPC, we directly set a new incoming ctx // instead of the regular outgoing context in the real gRPC connection. ctx := context.Background() - outputCh := make(chan sourcepb.ReadResponse) + outputCh := make(chan *sourcepb.ReadResponse) result := make([]*sourcepb.ReadResponse, 0) var readFnStream sourcepb.Source_ReadFnServer if tt.expectedErr { readFnStream = NewReadFnServerErrTest(ctx) } else { - readFnStream = NewReadFnServerTest(ctx, outputCh) + handshakeRequest := &sourcepb.ReadRequest{ + Handshake: &sourcepb.Handshake{ + Sot: true, + }, + } + readRequest := &sourcepb.ReadRequest{ + Request: &sourcepb.ReadRequest_Request{ + NumRecords: 1, + TimeoutInMs: 1000, + }, + } + readFnStream = NewReadFnServerTest(ctx, outputCh, []*sourcepb.ReadRequest{handshakeRequest, readRequest}) } var wg sync.WaitGroup @@ -171,11 +246,11 @@ func TestService_ReadFn(t *testing.T) { go func() { defer wg.Done() for msg := range outputCh { - result = append(result, &msg) + result = append(result, msg) } }() - err := fs.ReadFn(tt.input, readFnStream) + err := fs.ReadFn(readFnStream) close(outputCh) wg.Wait() @@ -196,20 +271,36 @@ func TestService_ReadFn(t *testing.T) { func TestService_AckFn(t *testing.T) { fs := &Service{Source: TestSource{}} ctx := context.Background() - got, err := fs.AckFn(ctx, &sourcepb.AckRequest{ - Request: &sourcepb.AckRequest_Request{ - Offsets: []*sourcepb.Offset{ - { - PartitionId: 0, - Offset: []byte("test"), - }, - }, + offsets := []*sourcepb.Offset{ + { + PartitionId: 0, + Offset: []byte("test"), }, - }) - assert.Equal(t, got, &sourcepb.AckResponse{ - Result: &sourcepb.AckResponse_Result{}, - }) + } + ackFnStream := NewAckFnServerTest(ctx, offsets) + + err := fs.AckFn(ackFnStream) assert.NoError(t, err) + + expectedResponses := []*sourcepb.AckResponse{ + { + Result: &sourcepb.AckResponse_Result{ + Success: &emptypb.Empty{}, + }, + Handshake: &sourcepb.Handshake{ + Sot: true, + }, + }, + { + Result: &sourcepb.AckResponse_Result{ + Success: &emptypb.Empty{}, + }, + }, + } + + if !reflect.DeepEqual(ackFnStream.responses, expectedResponses) { + t.Errorf("AckFn() responses = %v, want %v", ackFnStream.responses, expectedResponses) + } } func TestService_PendingFn(t *testing.T) { diff --git a/pkg/sourcetransformer/examples/assign_event_time/go.mod b/pkg/sourcetransformer/examples/assign_event_time/go.mod index 50a17e14..54232c8a 100644 --- a/pkg/sourcetransformer/examples/assign_event_time/go.mod +++ b/pkg/sourcetransformer/examples/assign_event_time/go.mod @@ -1,17 +1,17 @@ module assign_event_time -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. -require github.com/numaproj/numaflow-go v0.8.0 +require github.com/numaproj/numaflow-go v0.9.0 require ( - github.com/golang/protobuf v1.5.3 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect ) diff --git a/pkg/sourcetransformer/examples/assign_event_time/go.sum b/pkg/sourcetransformer/examples/assign_event_time/go.sum index 95c8479a..36997a49 100644 --- a/pkg/sourcetransformer/examples/assign_event_time/go.sum +++ b/pkg/sourcetransformer/examples/assign_event_time/go.sum @@ -1,24 +1,24 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sourcetransformer/examples/event_time_filter/go.mod b/pkg/sourcetransformer/examples/event_time_filter/go.mod index 156b9909..b7ca4732 100644 --- a/pkg/sourcetransformer/examples/event_time_filter/go.mod +++ b/pkg/sourcetransformer/examples/event_time_filter/go.mod @@ -1,25 +1,25 @@ module event_time_filter -go 1.20 +go 1.21 replace github.com/numaproj/numaflow-go => ../../../.. require ( - github.com/numaproj/numaflow-go v0.8.0 - github.com/stretchr/testify v1.8.1 + github.com/numaproj/numaflow-go v0.9.0 + github.com/stretchr/testify v1.9.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/grpc v1.57.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect + google.golang.org/grpc v1.66.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/pkg/sourcetransformer/examples/event_time_filter/go.sum b/pkg/sourcetransformer/examples/event_time_filter/go.sum index 3db6e45f..15b67493 100644 --- a/pkg/sourcetransformer/examples/event_time_filter/go.sum +++ b/pkg/sourcetransformer/examples/event_time_filter/go.sum @@ -1,43 +1,34 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= +google.golang.org/grpc v1.66.0 h1:DibZuoBznOxbDQxRINckZcUvnCEvrW9pcWIE2yF9r1c= +google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/sourcetransformer/examples/event_time_filter/impl/filter_test.go b/pkg/sourcetransformer/examples/event_time_filter/impl/filter_test.go index fb8b5b06..2ddd5cd7 100644 --- a/pkg/sourcetransformer/examples/event_time_filter/impl/filter_test.go +++ b/pkg/sourcetransformer/examples/event_time_filter/impl/filter_test.go @@ -23,6 +23,10 @@ func (d beforeYear2022Datum) Watermark() time.Time { return time.Now() } +func (d beforeYear2022Datum) Headers() map[string]string { + return map[string]string{} +} + type withinYear2022Datum struct{} func (d withinYear2022Datum) ID() string { @@ -39,6 +43,9 @@ func (d withinYear2022Datum) EventTime() time.Time { func (d withinYear2022Datum) Watermark() time.Time { return time.Now() } +func (d withinYear2022Datum) Headers() map[string]string { + return map[string]string{} +} type afterYear2022Datum struct{} @@ -52,6 +59,9 @@ func (d afterYear2022Datum) EventTime() time.Time { func (d afterYear2022Datum) Watermark() time.Time { return time.Now() } +func (d afterYear2022Datum) Headers() map[string]string { + return map[string]string{} +} func Test_FilterEventTime(t *testing.T) { testKeys := []string{"test-key"} diff --git a/pkg/sourcetransformer/server.go b/pkg/sourcetransformer/server.go index 7f22989a..b0aab9f5 100644 --- a/pkg/sourcetransformer/server.go +++ b/pkg/sourcetransformer/server.go @@ -3,6 +3,7 @@ package sourcetransformer import ( "context" "fmt" + "log" "os/signal" "sync" "syscall" @@ -49,8 +50,10 @@ func (m *server) Start(ctx context.Context) error { defer stop() // write server info to the file + serverInfo := info.GetDefaultServerInfo() + serverInfo.MinimumNumaflowVersion = info.MinimumNumaflowVersion[info.Sourcetransformer] // start listening on unix domain socket - lis, err := shared.PrepareServer(m.opts.sockAddr, m.opts.serverInfoFilePath, info.GetDefaultServerInfo()) + lis, err := shared.PrepareServer(m.opts.sockAddr, m.opts.serverInfoFilePath, serverInfo) if err != nil { return fmt.Errorf("failed to execute net.Listen(%q, %q): %v", uds, address, err) } @@ -71,6 +74,7 @@ func (m *server) Start(ctx context.Context) error { defer wg.Done() select { case <-m.shutdownCh: + log.Printf("shutdown signal received") case <-ctxWithSignal.Done(): } shared.StopGRPCServer(m.grpcServer) diff --git a/pkg/sourcetransformer/service.go b/pkg/sourcetransformer/service.go index 1602a50a..e19690f3 100644 --- a/pkg/sourcetransformer/service.go +++ b/pkg/sourcetransformer/service.go @@ -2,6 +2,15 @@ package sourcetransformer import ( "context" + "errors" + "fmt" + "io" + "log" + "runtime/debug" + + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" @@ -29,29 +38,150 @@ func (fs *Service) IsReady(context.Context, *emptypb.Empty) (*v1.ReadyResponse, return &v1.ReadyResponse{Ready: true}, nil } +var errTransformerPanic = errors.New("transformer function panicked") + +// recvWithContext wraps stream.Recv() to respect context cancellation. +func recvWithContext(ctx context.Context, stream v1.SourceTransform_SourceTransformFnServer) (*v1.SourceTransformRequest, error) { + type recvResult struct { + req *v1.SourceTransformRequest + err error + } + + resultCh := make(chan recvResult, 1) + go func() { + req, err := stream.Recv() + resultCh <- recvResult{req: req, err: err} + }() + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case result := <-resultCh: + return result.req, result.err + } +} + // SourceTransformFn applies a function to each request element. // In addition to map function, SourceTransformFn also supports assigning a new event time to response. // SourceTransformFn can be used only at source vertex by source data transformer. -func (fs *Service) SourceTransformFn(ctx context.Context, d *v1.SourceTransformRequest) (*v1.SourceTransformResponse, error) { - // handle panic +func (fs *Service) SourceTransformFn(stream v1.SourceTransform_SourceTransformFnServer) error { + + // perform handshake with client before processing requests + if err := fs.performHandshake(stream); err != nil { + return err + } + + ctx, cancel := context.WithCancel(stream.Context()) + defer cancel() + + // Use error group to manage goroutines, the groupCtx is cancelled when any of the + // goroutines return an error for the first time or the first time the wait returns. + grp, groupCtx := errgroup.WithContext(ctx) + + senderCh := make(chan *v1.SourceTransformResponse, 500) // FIXME: identify the right buffer size + // goroutine to send the responses back to the client + grp.Go(func() error { + for { + select { + case <-groupCtx.Done(): + return groupCtx.Err() + case resp := <-senderCh: + if err := stream.Send(resp); err != nil { + log.Printf("Failed to send response: %v", err) + return fmt.Errorf("failed to send response to client: %w", err) + } + } + } + }) + + var readErr error +outer: + for { + d, err := recvWithContext(groupCtx, stream) + if errors.Is(err, context.Canceled) { + log.Printf("Context cancelled, stopping the SourceTransformFn") + break outer + } + if errors.Is(err, io.EOF) { + log.Printf("EOF received, stopping the SourceTransformFn") + break outer + } + if err != nil { + log.Printf("Failed to receive request: %v", err) + readErr = err + // read loop is not part of the error group, so we need to cancel the context + // to signal the other goroutines to stop processing. + cancel() + break outer + } + grp.Go(func() (err error) { + return fs.handleRequest(groupCtx, d, senderCh) + }) + } + + // wait for all the goroutines to finish, if any of the goroutines return an error, wait will return that error immediately. + if err := grp.Wait(); err != nil { + log.Printf("Stopping the SourceTransformFn with err, %s", err) + fs.shutdownCh <- struct{}{} + return status.Errorf(codes.Internal, err.Error()) + } + + // check if there was an error while reading from the stream + if readErr != nil { + return status.Errorf(codes.Internal, readErr.Error()) + } + return nil +} + +// performHandshake handles the handshake logic at the start of the stream. +func (fs *Service) performHandshake(stream v1.SourceTransform_SourceTransformFnServer) error { + req, err := stream.Recv() + if err != nil { + return status.Errorf(codes.Internal, "failed to receive handshake: %v", err) + } + if req.GetHandshake() == nil || !req.GetHandshake().GetSot() { + return status.Errorf(codes.InvalidArgument, "invalid handshake") + } + handshakeResponse := &v1.SourceTransformResponse{ + Handshake: &v1.Handshake{ + Sot: true, + }, + } + if err := stream.Send(handshakeResponse); err != nil { + return fmt.Errorf("sending handshake response to client over gRPC stream: %w", err) + } + return nil +} + +// handleRequest processes each request and sends the response to the response channel. +func (fs *Service) handleRequest(ctx context.Context, req *v1.SourceTransformRequest, responseCh chan<- *v1.SourceTransformResponse) (err error) { defer func() { if r := recover(); r != nil { - fs.shutdownCh <- struct{}{} + log.Printf("panic inside handler: %v %v", r, string(debug.Stack())) + err = errTransformerPanic } }() - var hd = NewHandlerDatum(d.GetValue(), d.EventTime.AsTime(), d.Watermark.AsTime(), d.Headers) - messageTs := fs.Transformer.Transform(ctx, d.GetKeys(), hd) - var results []*v1.SourceTransformResponse_Result - for _, m := range messageTs.Items() { - results = append(results, &v1.SourceTransformResponse_Result{ - EventTime: timestamppb.New(m.EventTime()), + + request := req.GetRequest() + hd := NewHandlerDatum(request.GetValue(), request.GetEventTime().AsTime(), request.GetWatermark().AsTime(), request.GetHeaders()) + messages := fs.Transformer.Transform(ctx, request.GetKeys(), hd) + var elements []*v1.SourceTransformResponse_Result + for _, m := range messages.Items() { + elements = append(elements, &v1.SourceTransformResponse_Result{ Keys: m.Keys(), Value: m.Value(), Tags: m.Tags(), + EventTime: timestamppb.New(m.EventTime()), }) } - responseList := &v1.SourceTransformResponse{ - Results: results, + resp := &v1.SourceTransformResponse{ + Results: elements, + Id: req.GetRequest().GetId(), + } + select { + case responseCh <- resp: + case <-ctx.Done(): + return ctx.Err() } - return responseList, nil + return nil } diff --git a/pkg/sourcetransformer/service_test.go b/pkg/sourcetransformer/service_test.go index d57d937e..79449c9a 100644 --- a/pkg/sourcetransformer/service_test.go +++ b/pkg/sourcetransformer/service_test.go @@ -2,28 +2,83 @@ package sourcetransformer import ( "context" - "reflect" + "errors" + "fmt" + "net" "testing" "time" - "google.golang.org/protobuf/types/known/timestamppb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" - stpb "github.com/numaproj/numaflow-go/pkg/apis/proto/sourcetransform/v1" + proto "github.com/numaproj/numaflow-go/pkg/apis/proto/sourcetransform/v1" + "google.golang.org/protobuf/types/known/timestamppb" ) +func newTestServer(t *testing.T, register func(server *grpc.Server)) *grpc.ClientConn { + lis := bufconn.Listen(1024 * 1024) + t.Cleanup(func() { + _ = lis.Close() + }) + + server := grpc.NewServer() + t.Cleanup(func() { + server.Stop() + }) + + register(server) + + errChan := make(chan error, 1) + go func() { + // t.Fatal should only be called from the goroutine running the test + if err := server.Serve(lis); err != nil { + errChan <- err + } + }() + + dialer := func(context.Context, string) (net.Conn, error) { + return lis.Dial() + } + + conn, err := grpc.NewClient("passthrough://", grpc.WithContextDialer(dialer), grpc.WithTransportCredentials(insecure.NewCredentials())) + t.Cleanup(func() { + _ = conn.Close() + }) + if err != nil { + t.Fatalf("Creating new gRPC client connection: %v", err) + } + + var grpcServerErr error + select { + case grpcServerErr = <-errChan: + case <-time.After(500 * time.Millisecond): + grpcServerErr = errors.New("gRPC server didn't start in 500ms") + } + if err != nil { + t.Fatalf("Failed to start gRPC server: %v", grpcServerErr) + } + + return conn +} + +var testTime = time.Date(2021, 8, 15, 14, 30, 45, 100, time.Local) + func TestService_sourceTransformFn(t *testing.T) { type args struct { ctx context.Context - d *stpb.SourceTransformRequest + d *proto.SourceTransformRequest } - testTime := time.Date(2021, 8, 15, 14, 30, 45, 100, time.Local) tests := []struct { name string handler SourceTransformer args args - want *stpb.SourceTransformResponse - wantErr bool + want *proto.SourceTransformResponse }{ { name: "sourceTransform_fn_forward_msg", @@ -33,15 +88,17 @@ func TestService_sourceTransformFn(t *testing.T) { }), args: args{ ctx: context.Background(), - d: &stpb.SourceTransformRequest{ - Keys: []string{"client"}, - Value: []byte(`test`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + d: &proto.SourceTransformRequest{ + Request: &proto.SourceTransformRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, }, }, - want: &stpb.SourceTransformResponse{ - Results: []*stpb.SourceTransformResponse_Result{ + want: &proto.SourceTransformResponse{ + Results: []*proto.SourceTransformResponse_Result{ { EventTime: timestamppb.New(testTime), Keys: []string{"client_test"}, @@ -49,7 +106,6 @@ func TestService_sourceTransformFn(t *testing.T) { }, }, }, - wantErr: false, }, { name: "sourceTransform_fn_forward_msg_forward_to_all", @@ -59,22 +115,23 @@ func TestService_sourceTransformFn(t *testing.T) { }), args: args{ ctx: context.Background(), - d: &stpb.SourceTransformRequest{ - Keys: []string{"client"}, - Value: []byte(`test`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + d: &proto.SourceTransformRequest{ + Request: &proto.SourceTransformRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, }, }, - want: &stpb.SourceTransformResponse{ - Results: []*stpb.SourceTransformResponse_Result{ + want: &proto.SourceTransformResponse{ + Results: []*proto.SourceTransformResponse_Result{ { EventTime: timestamppb.New(testTime), Value: []byte(`test`), }, }, }, - wantErr: false, }, { name: "sourceTransform_fn_forward_msg_drop_msg", @@ -83,42 +140,159 @@ func TestService_sourceTransformFn(t *testing.T) { }), args: args{ ctx: context.Background(), - d: &stpb.SourceTransformRequest{ - Keys: []string{"client"}, - Value: []byte(`test`), - EventTime: timestamppb.New(time.Time{}), - Watermark: timestamppb.New(time.Time{}), + d: &proto.SourceTransformRequest{ + Request: &proto.SourceTransformRequest_Request{ + Keys: []string{"client"}, + Value: []byte(`test`), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, }, }, - want: &stpb.SourceTransformResponse{ - Results: []*stpb.SourceTransformResponse_Result{ + want: &proto.SourceTransformResponse{ + Results: []*proto.SourceTransformResponse_Result{ { EventTime: timestamppb.New(testTime), Tags: []string{DROP}, - Value: []byte{}, + Value: nil, }, }, }, - wantErr: false, }, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fs := &Service{ + svc := &Service{ Transformer: tt.handler, } - // here's a trick for testing: - // because we are not using gRPC, we directly set a new incoming ctx - // instead of the regular outgoing context in the real gRPC connection. - ctx := context.Background() - got, err := fs.SourceTransformFn(ctx, tt.args.d) - if (err != nil) != tt.wantErr { - t.Errorf("SourceTransformFn() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("SourceTransformFn() got = %v, want %v", got, tt.want) - } + + conn := newTestServer(t, func(server *grpc.Server) { + proto.RegisterSourceTransformServer(server, svc) + }) + + client := proto.NewSourceTransformClient(conn) + stream, err := client.SourceTransformFn(context.Background()) + require.NoError(t, err, "Creating stream") + + doHandshake(t, stream) + + err = stream.Send(tt.args.d) + require.NoError(t, err, "Sending message over the stream") + + got, err := stream.Recv() + require.NoError(t, err, "Receiving message from the stream") + + assert.Equal(t, got.Results, tt.want.Results) }) } } + +func doHandshake(t *testing.T, stream proto.SourceTransform_SourceTransformFnClient) { + t.Helper() + handshakeReq := &proto.SourceTransformRequest{ + Handshake: &proto.Handshake{Sot: true}, + } + err := stream.Send(handshakeReq) + require.NoError(t, err, "Sending handshake request to the stream") + + handshakeResp, err := stream.Recv() + require.NoError(t, err, "Receiving handshake response") + + require.Empty(t, handshakeResp.Results, "Invalid handshake response") + require.Empty(t, handshakeResp.Id, "Invalid handshake response") + require.NotNil(t, handshakeResp.Handshake, "Invalid handshake response") + require.True(t, handshakeResp.Handshake.Sot, "Invalid handshake response") +} + +func TestService_SourceTransformFn_Multiple_Messages(t *testing.T) { + svc := &Service{ + Transformer: SourceTransformFunc(func(ctx context.Context, keys []string, datum Datum) Messages { + msg := datum.Value() + return MessagesBuilder().Append(NewMessage(msg, testTime).WithKeys([]string{keys[0] + "_test"})) + }), + } + conn := newTestServer(t, func(server *grpc.Server) { + proto.RegisterSourceTransformServer(server, svc) + }) + + client := proto.NewSourceTransformClient(conn) + stream, err := client.SourceTransformFn(context.Background()) + require.NoError(t, err, "Creating stream") + + doHandshake(t, stream) + + const msgCount = 10 + for i := 0; i < msgCount; i++ { + msg := proto.SourceTransformRequest{ + Request: &proto.SourceTransformRequest_Request{ + Keys: []string{"client"}, + Value: []byte(fmt.Sprintf("test_%d", i)), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + } + err = stream.Send(&msg) + require.NoError(t, err, "Sending message over the stream") + } + err = stream.CloseSend() + require.NoError(t, err, "Closing the send direction of the stream") + + expectedResults := make([][]*proto.SourceTransformResponse_Result, msgCount) + for i := 0; i < msgCount; i++ { + expectedResults[i] = []*proto.SourceTransformResponse_Result{ + { + EventTime: timestamppb.New(testTime), + Keys: []string{"client_test"}, + Value: []byte(fmt.Sprintf("test_%d", i)), + }, + } + } + + results := make([][]*proto.SourceTransformResponse_Result, msgCount) + for i := 0; i < msgCount; i++ { + got, err := stream.Recv() + require.NoError(t, err, "Receiving message from the stream") + results[i] = got.Results + } + require.ElementsMatch(t, results, expectedResults) +} + +func TestService_SourceTransformFn_Panic(t *testing.T) { + svc := &Service{ + Transformer: SourceTransformFunc(func(ctx context.Context, keys []string, datum Datum) Messages { + panic("transformer panicked") + }), + // panic in the transformer causes the server to send a shutdown signal to shutdownCh channel. + // The function that errgroup runs in a goroutine will be blocked until this shutdown signal is received somewhere else. + // Since we don't listen for shutdown signal in the tests, we use buffered channel to unblock the server function. + shutdownCh: make(chan<- struct{}, 1), + } + conn := newTestServer(t, func(server *grpc.Server) { + proto.RegisterSourceTransformServer(server, svc) + }) + + client := proto.NewSourceTransformClient(conn) + stream, err := client.SourceTransformFn(context.Background()) + require.NoError(t, err, "Creating stream") + + doHandshake(t, stream) + + msg := proto.SourceTransformRequest{ + Request: &proto.SourceTransformRequest_Request{ + Keys: []string{"client"}, + Value: []byte("test"), + EventTime: timestamppb.New(time.Time{}), + Watermark: timestamppb.New(time.Time{}), + }, + } + err = stream.Send(&msg) + require.NoError(t, err, "Sending message over the stream") + err = stream.CloseSend() + require.NoError(t, err, "Closing the send direction of the stream") + _, err = stream.Recv() + require.Error(t, err, "Expected error while receiving message from the stream") + gotStatus, _ := status.FromError(err) + expectedStatus := status.Convert(status.Errorf(codes.Internal, errTransformerPanic.Error())) + require.Equal(t, expectedStatus, gotStatus) +} diff --git a/releases.md b/releases.md index ad958c86..ba0977e7 100644 --- a/releases.md +++ b/releases.md @@ -4,7 +4,7 @@ This document explains the release process for the Go SDK. You can find the most ### Before Release -If the version to be released has backwards incompatible changes, i.e. it does not support older versions of the Numaflow platform, +If the version to be released has backwards incompatible changes, i.e., it does not support older versions of the Numaflow platform, you must update the `MinimumNumaflowVersion` constant in the `pkg/info/types.go` file to the minimum Numaflow version that is supported by your new SDK version. Ensure that this change is merged and included in the release.