diff --git a/pkg/apis/proto/map/v1/map.pb.go b/pkg/apis/proto/map/v1/map.pb.go index 7d615a9b..58f978de 100644 --- a/pkg/apis/proto/map/v1/map.pb.go +++ b/pkg/apis/proto/map/v1/map.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 +// protoc-gen-go v1.33.0 // protoc v4.25.1 // source: pkg/apis/proto/map/v1/map.proto diff --git a/pkg/apis/proto/mapstream/v1/mapstream.pb.go b/pkg/apis/proto/mapstream/v1/mapstream.pb.go index 2a2f2bd1..6a276ae2 100644 --- a/pkg/apis/proto/mapstream/v1/mapstream.pb.go +++ b/pkg/apis/proto/mapstream/v1/mapstream.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 +// protoc-gen-go v1.33.0 // protoc v4.25.1 // source: pkg/apis/proto/mapstream/v1/mapstream.proto diff --git a/pkg/apis/proto/reduce/v1/reduce.pb.go b/pkg/apis/proto/reduce/v1/reduce.pb.go index 5376e90a..597d4c7c 100644 --- a/pkg/apis/proto/reduce/v1/reduce.pb.go +++ b/pkg/apis/proto/reduce/v1/reduce.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 +// protoc-gen-go v1.33.0 // protoc v4.25.1 // source: pkg/apis/proto/reduce/v1/reduce.proto diff --git a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go index 79cd3f65..fe19a9ae 100644 --- a/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go +++ b/pkg/apis/proto/sessionreduce/v1/sessionreduce.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 +// protoc-gen-go v1.33.0 // protoc v4.25.1 // source: pkg/apis/proto/sessionreduce/v1/sessionreduce.proto diff --git a/pkg/apis/proto/sideinput/v1/sideinput.pb.go b/pkg/apis/proto/sideinput/v1/sideinput.pb.go index 5fd5b7cd..a117b3a1 100644 --- a/pkg/apis/proto/sideinput/v1/sideinput.pb.go +++ b/pkg/apis/proto/sideinput/v1/sideinput.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 +// protoc-gen-go v1.33.0 // protoc v4.25.1 // source: pkg/apis/proto/sideinput/v1/sideinput.proto diff --git a/pkg/apis/proto/sink/v1/sink.pb.go b/pkg/apis/proto/sink/v1/sink.pb.go index 0bc0a086..3f0dae1f 100644 --- a/pkg/apis/proto/sink/v1/sink.pb.go +++ b/pkg/apis/proto/sink/v1/sink.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 +// protoc-gen-go v1.33.0 // protoc v4.25.1 // source: pkg/apis/proto/sink/v1/sink.proto @@ -22,6 +22,56 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// Status is the status of the response. +type Status int32 + +const ( + Status_SUCCESS Status = 0 + Status_FAILURE Status = 1 + Status_FALLBACK Status = 2 +) + +// Enum value maps for Status. +var ( + Status_name = map[int32]string{ + 0: "SUCCESS", + 1: "FAILURE", + 2: "FALLBACK", + } + Status_value = map[string]int32{ + "SUCCESS": 0, + "FAILURE": 1, + "FALLBACK": 2, + } +) + +func (x Status) Enum() *Status { + p := new(Status) + *p = x + return p +} + +func (x Status) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Status) Descriptor() protoreflect.EnumDescriptor { + return file_pkg_apis_proto_sink_v1_sink_proto_enumTypes[0].Descriptor() +} + +func (Status) Type() protoreflect.EnumType { + return &file_pkg_apis_proto_sink_v1_sink_proto_enumTypes[0] +} + +func (x Status) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Status.Descriptor instead. +func (Status) EnumDescriptor() ([]byte, []int) { + return file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP(), []int{0} +} + // * // SinkRequest represents a request element. type SinkRequest struct { @@ -216,8 +266,8 @@ type SinkResponse_Result struct { // id is the ID of the message, can be used to uniquely identify the message. Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - // success denotes the status of persisting to disk. if set to false, it means writing to sink for the message failed. - Success bool `protobuf:"varint,2,opt,name=success,proto3" json:"success,omitempty"` + // status denotes the status of persisting to sink. It can be SUCCESS, FAILURE, or FALLBACK. + Status Status `protobuf:"varint,2,opt,name=status,proto3,enum=sink.v1.Status" json:"status,omitempty"` // err_msg is the error message, set it if success is set to false. ErrMsg string `protobuf:"bytes,3,opt,name=err_msg,json=errMsg,proto3" json:"err_msg,omitempty"` } @@ -261,11 +311,11 @@ func (x *SinkResponse_Result) GetId() string { return "" } -func (x *SinkResponse_Result) GetSuccess() bool { +func (x *SinkResponse_Result) GetStatus() Status { if x != nil { - return x.Success + return x.Status } - return false + return Status_SUCCESS } func (x *SinkResponse_Result) GetErrMsg() string { @@ -306,28 +356,32 @@ var file_pkg_apis_proto_sink_v1_sink_proto_rawDesc = []byte{ 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x25, 0x0a, 0x0d, 0x52, 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x22, 0x93, 0x01, 0x0a, 0x0c, 0x53, 0x69, + 0x28, 0x08, 0x52, 0x05, 0x72, 0x65, 0x61, 0x64, 0x79, 0x22, 0xa2, 0x01, 0x0a, 0x0c, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, 0x73, 0x75, 0x6c, - 0x74, 0x73, 0x1a, 0x4b, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x0e, 0x0a, 0x02, - 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, - 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x73, - 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, - 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x32, - 0x7a, 0x0a, 0x04, 0x53, 0x69, 0x6e, 0x6b, 0x12, 0x37, 0x0a, 0x06, 0x53, 0x69, 0x6e, 0x6b, 0x46, - 0x6e, 0x12, 0x14, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x6e, 0x6b, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, - 0x31, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, 0x01, - 0x12, 0x39, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x65, - 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x38, 0x5a, 0x36, 0x67, - 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, 0x72, - 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, 0x70, - 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, 0x69, - 0x6e, 0x6b, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x74, 0x73, 0x1a, 0x5a, 0x0a, 0x06, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x27, 0x0a, 0x06, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x73, + 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x2a, 0x30, + 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x43, 0x43, + 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, + 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x46, 0x41, 0x4c, 0x4c, 0x42, 0x41, 0x43, 0x4b, 0x10, 0x02, + 0x32, 0x7a, 0x0a, 0x04, 0x53, 0x69, 0x6e, 0x6b, 0x12, 0x37, 0x0a, 0x06, 0x53, 0x69, 0x6e, 0x6b, + 0x46, 0x6e, 0x12, 0x14, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x6e, + 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, + 0x76, 0x31, 0x2e, 0x53, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x28, + 0x01, 0x12, 0x39, 0x0a, 0x07, 0x49, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x12, 0x16, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x16, 0x2e, 0x73, 0x69, 0x6e, 0x6b, 0x2e, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x61, 0x64, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x38, 0x5a, 0x36, + 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x70, + 0x72, 0x6f, 0x6a, 0x2f, 0x6e, 0x75, 0x6d, 0x61, 0x66, 0x6c, 0x6f, 0x77, 0x2d, 0x67, 0x6f, 0x2f, + 0x70, 0x6b, 0x67, 0x2f, 0x61, 0x70, 0x69, 0x73, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x73, + 0x69, 0x6e, 0x6b, 0x2f, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -342,30 +396,33 @@ func file_pkg_apis_proto_sink_v1_sink_proto_rawDescGZIP() []byte { return file_pkg_apis_proto_sink_v1_sink_proto_rawDescData } +var file_pkg_apis_proto_sink_v1_sink_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_pkg_apis_proto_sink_v1_sink_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_pkg_apis_proto_sink_v1_sink_proto_goTypes = []interface{}{ - (*SinkRequest)(nil), // 0: sink.v1.SinkRequest - (*ReadyResponse)(nil), // 1: sink.v1.ReadyResponse - (*SinkResponse)(nil), // 2: sink.v1.SinkResponse - nil, // 3: sink.v1.SinkRequest.HeadersEntry - (*SinkResponse_Result)(nil), // 4: sink.v1.SinkResponse.Result - (*timestamppb.Timestamp)(nil), // 5: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 6: google.protobuf.Empty + (Status)(0), // 0: sink.v1.Status + (*SinkRequest)(nil), // 1: sink.v1.SinkRequest + (*ReadyResponse)(nil), // 2: sink.v1.ReadyResponse + (*SinkResponse)(nil), // 3: sink.v1.SinkResponse + nil, // 4: sink.v1.SinkRequest.HeadersEntry + (*SinkResponse_Result)(nil), // 5: sink.v1.SinkResponse.Result + (*timestamppb.Timestamp)(nil), // 6: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 7: google.protobuf.Empty } var file_pkg_apis_proto_sink_v1_sink_proto_depIdxs = []int32{ - 5, // 0: sink.v1.SinkRequest.event_time:type_name -> google.protobuf.Timestamp - 5, // 1: sink.v1.SinkRequest.watermark:type_name -> google.protobuf.Timestamp - 3, // 2: sink.v1.SinkRequest.headers:type_name -> sink.v1.SinkRequest.HeadersEntry - 4, // 3: sink.v1.SinkResponse.results:type_name -> sink.v1.SinkResponse.Result - 0, // 4: sink.v1.Sink.SinkFn:input_type -> sink.v1.SinkRequest - 6, // 5: sink.v1.Sink.IsReady:input_type -> google.protobuf.Empty - 2, // 6: sink.v1.Sink.SinkFn:output_type -> sink.v1.SinkResponse - 1, // 7: sink.v1.Sink.IsReady:output_type -> sink.v1.ReadyResponse - 6, // [6:8] is the sub-list for method output_type - 4, // [4:6] is the sub-list for method input_type - 4, // [4:4] is the sub-list for extension type_name - 4, // [4:4] is the sub-list for extension extendee - 0, // [0:4] is the sub-list for field type_name + 6, // 0: sink.v1.SinkRequest.event_time:type_name -> google.protobuf.Timestamp + 6, // 1: sink.v1.SinkRequest.watermark:type_name -> google.protobuf.Timestamp + 4, // 2: sink.v1.SinkRequest.headers:type_name -> sink.v1.SinkRequest.HeadersEntry + 5, // 3: sink.v1.SinkResponse.results:type_name -> sink.v1.SinkResponse.Result + 0, // 4: sink.v1.SinkResponse.Result.status:type_name -> sink.v1.Status + 1, // 5: sink.v1.Sink.SinkFn:input_type -> sink.v1.SinkRequest + 7, // 6: sink.v1.Sink.IsReady:input_type -> google.protobuf.Empty + 3, // 7: sink.v1.Sink.SinkFn:output_type -> sink.v1.SinkResponse + 2, // 8: sink.v1.Sink.IsReady:output_type -> sink.v1.ReadyResponse + 7, // [7:9] is the sub-list for method output_type + 5, // [5:7] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_pkg_apis_proto_sink_v1_sink_proto_init() } @@ -428,13 +485,14 @@ func file_pkg_apis_proto_sink_v1_sink_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_pkg_apis_proto_sink_v1_sink_proto_rawDesc, - NumEnums: 0, + NumEnums: 1, NumMessages: 5, NumExtensions: 0, NumServices: 1, }, GoTypes: file_pkg_apis_proto_sink_v1_sink_proto_goTypes, DependencyIndexes: file_pkg_apis_proto_sink_v1_sink_proto_depIdxs, + EnumInfos: file_pkg_apis_proto_sink_v1_sink_proto_enumTypes, MessageInfos: file_pkg_apis_proto_sink_v1_sink_proto_msgTypes, }.Build() File_pkg_apis_proto_sink_v1_sink_proto = out.File diff --git a/pkg/apis/proto/sink/v1/sink.proto b/pkg/apis/proto/sink/v1/sink.proto index 5e47fe4c..0bdd5a27 100644 --- a/pkg/apis/proto/sink/v1/sink.proto +++ b/pkg/apis/proto/sink/v1/sink.proto @@ -35,6 +35,15 @@ message ReadyResponse { bool ready = 1; } +/* + * Status is the status of the response. + */ +enum Status { + SUCCESS = 0; + FAILURE = 1; + FALLBACK = 2; +} + /** * SinkResponse is the individual response of each message written to the sink. */ @@ -42,8 +51,8 @@ message SinkResponse { message Result { // id is the ID of the message, can be used to uniquely identify the message. string id = 1; - // success denotes the status of persisting to disk. if set to false, it means writing to sink for the message failed. - bool success = 2; + // status denotes the status of persisting to sink. It can be SUCCESS, FAILURE, or FALLBACK. + Status status = 2; // err_msg is the error message, set it if success is set to false. string err_msg = 3; } diff --git a/pkg/apis/proto/source/v1/source.pb.go b/pkg/apis/proto/source/v1/source.pb.go index 1304f4ee..7927e25e 100644 --- a/pkg/apis/proto/source/v1/source.pb.go +++ b/pkg/apis/proto/source/v1/source.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 +// protoc-gen-go v1.33.0 // protoc v4.25.1 // source: pkg/apis/proto/source/v1/source.proto diff --git a/pkg/apis/proto/sourcetransform/v1/transform.pb.go b/pkg/apis/proto/sourcetransform/v1/transform.pb.go index 3a34e351..b56efc20 100644 --- a/pkg/apis/proto/sourcetransform/v1/transform.pb.go +++ b/pkg/apis/proto/sourcetransform/v1/transform.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 +// protoc-gen-go v1.33.0 // protoc v4.25.1 // source: pkg/apis/proto/sourcetransform/v1/transform.proto diff --git a/pkg/sinker/examples/log/Makefile b/pkg/sinker/examples/log/Makefile index 0850b8dc..e94cbff6 100644 --- a/pkg/sinker/examples/log/Makefile +++ b/pkg/sinker/examples/log/Makefile @@ -10,7 +10,7 @@ image-push: build docker buildx build -t "quay.io/numaio/numaflow-go/sink-log:${TAG}" --platform linux/amd64,linux/arm64 --target log . --push .PHONY: image -imageimage: build +image: build docker build -t "quay.io/numaio/numaflow-go/sink-log:${TAG}" --target log . @if [ "$(PUSH)" = "true" ]; then docker push "quay.io/numaio/numaflow-go/sink-log:${TAG}"; fi diff --git a/pkg/sinker/examples/log/main.go b/pkg/sinker/examples/log/main.go index 89f18a76..1f7c1156 100644 --- a/pkg/sinker/examples/log/main.go +++ b/pkg/sinker/examples/log/main.go @@ -20,6 +20,8 @@ func (l *logSink) Sink(ctx context.Context, datumStreamCh <-chan sinksdk.Datum) fmt.Println("User Defined Sink:", string(d.Value())) id := d.ID() result = result.Append(sinksdk.ResponseOK(id)) + // if we are not able to write to sink and if we have a fallback sink configured + // we can use sinksdk.ResponseFallback(id)) to write the message to fallback sink } return result } diff --git a/pkg/sinker/options.go b/pkg/sinker/options.go index a913ae8b..361cc2e5 100644 --- a/pkg/sinker/options.go +++ b/pkg/sinker/options.go @@ -1,5 +1,7 @@ package sinker +import "os" + type options struct { sockAddr string maxMessageSize int @@ -10,10 +12,19 @@ type options struct { type Option func(*options) func defaultOptions() *options { + defaultPath := serverInfoFilePath + defaultAddress := address + + // If the container type is fallback sink, then use the fallback sink address and path. + if os.Getenv(EnvUDContainerType) == UDContainerFallbackSink { + defaultPath = fbServerInfoFilePath + defaultAddress = fbAddress + } + return &options{ - sockAddr: address, + sockAddr: defaultAddress, maxMessageSize: defaultMaxMessageSize, - serverInfoFilePath: serverInfoFilePath, + serverInfoFilePath: defaultPath, } } diff --git a/pkg/sinker/service.go b/pkg/sinker/service.go index 4ac48c15..67f99838 100644 --- a/pkg/sinker/service.go +++ b/pkg/sinker/service.go @@ -12,10 +12,14 @@ import ( ) const ( - uds = "unix" - defaultMaxMessageSize = 1024 * 1024 * 64 // 64MB - address = "/var/run/numaflow/sink.sock" - serverInfoFilePath = "/var/run/numaflow/sinker-server-info" + uds = "unix" + defaultMaxMessageSize = 1024 * 1024 * 64 // 64MB + address = "/var/run/numaflow/sink.sock" + fbAddress = "/var/run/numaflow/fb-sink.sock" + serverInfoFilePath = "/var/run/numaflow/sinker-server-info" + fbServerInfoFilePath = "/var/run/numaflow/fb-sinker-server-info" + EnvUDContainerType = "NUMAFLOW_UD_CONTAINER_TYPE" + UDContainerFallbackSink = "fb-udsink" ) // handlerDatum implements the Datum interface and is used in the sink functions. @@ -78,11 +82,23 @@ func (fs *Service) SinkFn(stream sinkpb.Sink_SinkFnServer) error { defer wg.Done() messages := fs.Sinker.Sink(ctx, datumStreamCh) for _, msg := range messages { - resultList = append(resultList, &sinkpb.SinkResponse_Result{ - Id: msg.ID, - Success: msg.Success, - ErrMsg: msg.Err, - }) + if msg.Fallback { + resultList = append(resultList, &sinkpb.SinkResponse_Result{ + Id: msg.ID, + Status: sinkpb.Status_FALLBACK, + }) + } else if msg.Success { + resultList = append(resultList, &sinkpb.SinkResponse_Result{ + Id: msg.ID, + Status: sinkpb.Status_SUCCESS, + }) + } else { + resultList = append(resultList, &sinkpb.SinkResponse_Result{ + Id: msg.ID, + Status: sinkpb.Status_FAILURE, + ErrMsg: msg.Err, + }) + } } }() diff --git a/pkg/sinker/service_test.go b/pkg/sinker/service_test.go index 176499a6..d6ad1b5b 100644 --- a/pkg/sinker/service_test.go +++ b/pkg/sinker/service_test.go @@ -86,19 +86,19 @@ func TestService_SinkFn(t *testing.T) { }), expected: []*sinkpb.SinkResponse_Result{ { - Success: true, - Id: "one-processed", - ErrMsg: "", + Status: sinkpb.Status_SUCCESS, + Id: "one-processed", + ErrMsg: "", }, { - Success: true, - Id: "two-processed", - ErrMsg: "", + Status: sinkpb.Status_SUCCESS, + Id: "two-processed", + ErrMsg: "", }, { - Success: true, - Id: "three-processed", - ErrMsg: "", + Status: sinkpb.Status_SUCCESS, + Id: "three-processed", + ErrMsg: "", }, }, }, @@ -140,19 +140,19 @@ func TestService_SinkFn(t *testing.T) { }), expected: []*sinkpb.SinkResponse_Result{ { - Success: false, - Id: "one-processed", - ErrMsg: "unknown error", + Status: sinkpb.Status_FAILURE, + Id: "one-processed", + ErrMsg: "unknown error", }, { - Success: false, - Id: "two-processed", - ErrMsg: "unknown error", + Status: sinkpb.Status_FAILURE, + Id: "two-processed", + ErrMsg: "unknown error", }, { - Success: false, - Id: "three-processed", - ErrMsg: "unknown error", + Status: sinkpb.Status_FAILURE, + Id: "three-processed", + ErrMsg: "unknown error", }, }, }, diff --git a/pkg/sinker/types.go b/pkg/sinker/types.go index 3b33f47b..125b914c 100644 --- a/pkg/sinker/types.go +++ b/pkg/sinker/types.go @@ -8,6 +8,8 @@ type Response struct { Success bool `json:"success"` // Err represents the error message when "success" is false. Err string `json:"err,omitempty"` + // Fallback is true if the message to be sent to the fallback sink. + Fallback bool `json:"fallback,omitempty"` } type Responses []Response @@ -28,10 +30,20 @@ func (r Responses) Items() []Response { return r } +// ResponseOK creates a successful Response with the given id. +// The Success field is set to true. func ResponseOK(id string) Response { return Response{ID: id, Success: true} } +// ResponseFailure creates a failed Response with the given id and error message. +// The Success field is set to false and the Err field is set to the provided error message. func ResponseFailure(id, errMsg string) Response { return Response{ID: id, Success: false, Err: errMsg} } + +// ResponseFallback creates a Response with the Fallback field set to true. +// This indicates that the message should be sent to the fallback sink. +func ResponseFallback(id string) Response { + return Response{ID: id, Fallback: true} +}