diff --git a/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml b/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml index f2451c4094..0dd115f8d2 100644 --- a/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml +++ b/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml @@ -3934,16 +3934,22 @@ spec: type: array watermark: default: - propagate: false + disabled: false description: Watermark enables watermark progression across the entire pipeline. Updating this after the pipeline has been created will have no impact and will be ignored. To make the pipeline honor any changes to the setting, the pipeline should be recreated. properties: - propagate: + disabled: default: false - description: Propagate toggles the watermark propagation. + description: Disabled toggles the watermark propagation, defaults + to false. type: boolean + maxDelay: + default: 0s + description: Maximum delay allowed for watermark calculation, + defaults to "0s", which means no delay. + type: string type: object type: object status: diff --git a/config/install.yaml b/config/install.yaml index 364b925c11..888bf18351 100644 --- a/config/install.yaml +++ b/config/install.yaml @@ -8660,16 +8660,22 @@ spec: type: array watermark: default: - propagate: false + disabled: false description: Watermark enables watermark progression across the entire pipeline. Updating this after the pipeline has been created will have no impact and will be ignored. To make the pipeline honor any changes to the setting, the pipeline should be recreated. properties: - propagate: + disabled: default: false - description: Propagate toggles the watermark propagation. + description: Disabled toggles the watermark propagation, defaults + to false. type: boolean + maxDelay: + default: 0s + description: Maximum delay allowed for watermark calculation, + defaults to "0s", which means no delay. + type: string type: object type: object status: diff --git a/config/namespace-install.yaml b/config/namespace-install.yaml index a4acbaf9ee..266eb38f15 100644 --- a/config/namespace-install.yaml +++ b/config/namespace-install.yaml @@ -8660,16 +8660,22 @@ spec: type: array watermark: default: - propagate: false + disabled: false description: Watermark enables watermark progression across the entire pipeline. Updating this after the pipeline has been created will have no impact and will be ignored. To make the pipeline honor any changes to the setting, the pipeline should be recreated. properties: - propagate: + disabled: default: false - description: Propagate toggles the watermark propagation. + description: Disabled toggles the watermark propagation, defaults + to false. type: boolean + maxDelay: + default: 0s + description: Maximum delay allowed for watermark calculation, + defaults to "0s", which means no delay. + type: string type: object type: object status: diff --git a/controllers/vertex/controller.go b/controllers/vertex/controller.go index 1b9642c08a..c02e77d7fb 100644 --- a/controllers/vertex/controller.go +++ b/controllers/vertex/controller.go @@ -327,8 +327,17 @@ func (r *vertexReconciler) buildPodSpec(vertex *dfv1.Vertex, pl *dfv1.Pipeline, } podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, corev1.EnvVar{ - Name: dfv1.EnvWatermarkOn, - Value: fmt.Sprintf("%t", pl.Spec.Watermark.Propagate), + Name: dfv1.EnvWatermarkDisabled, + Value: fmt.Sprintf("%t", pl.Spec.Watermark.Disabled), + }) + + maxDelay := "0s" + if x := pl.Spec.Watermark.MaxDelay; x != nil { + maxDelay = x.Duration.String() + } + podSpec.Containers[0].Env = append(podSpec.Containers[0].Env, corev1.EnvVar{ + Name: dfv1.EnvWatermarkMaxDelay, + Value: maxDelay, }) return podSpec, nil diff --git a/controllers/vertex/controller_test.go b/controllers/vertex/controller_test.go index 4c3f10db32..04340d9638 100644 --- a/controllers/vertex/controller_test.go +++ b/controllers/vertex/controller_test.go @@ -185,6 +185,8 @@ func Test_BuildPodSpec(t *testing.T) { assert.Contains(t, envNames, dfv1.EnvISBSvcSentinelMaster) assert.Contains(t, envNames, dfv1.EnvISBSvcRedisUser) assert.Contains(t, envNames, dfv1.EnvISBSvcRedisURL) + assert.Contains(t, envNames, dfv1.EnvWatermarkDisabled) + assert.Contains(t, envNames, dfv1.EnvWatermarkMaxDelay) argStr := strings.Join(spec.InitContainers[0].Args, " ") assert.Contains(t, argStr, "--buffers=") for _, b := range testObj.GetToBuffers() { @@ -221,6 +223,8 @@ func Test_BuildPodSpec(t *testing.T) { assert.Contains(t, envNames, dfv1.EnvISBSvcSentinelMaster) assert.Contains(t, envNames, dfv1.EnvISBSvcRedisUser) assert.Contains(t, envNames, dfv1.EnvISBSvcRedisURL) + assert.Contains(t, envNames, dfv1.EnvWatermarkDisabled) + assert.Contains(t, envNames, dfv1.EnvWatermarkMaxDelay) argStr := strings.Join(spec.InitContainers[0].Args, " ") assert.Contains(t, argStr, "--buffers=") for _, b := range testObj.GetFromBuffers() { @@ -296,6 +300,8 @@ func Test_BuildPodSpec(t *testing.T) { assert.Contains(t, envNames, dfv1.EnvISBSvcRedisSentinelPassword) assert.Contains(t, envNames, dfv1.EnvISBSvcRedisUser) assert.Contains(t, envNames, dfv1.EnvISBSvcRedisURL) + assert.Contains(t, envNames, dfv1.EnvWatermarkDisabled) + assert.Contains(t, envNames, dfv1.EnvWatermarkMaxDelay) udfEnvNames := []string{} udfEnvValues := []string{} for _, e := range spec.Containers[1].Env { diff --git a/docs/APIs.md b/docs/APIs.md index 09064dc582..ba52abf3f4 100644 --- a/docs/APIs.md +++ b/docs/APIs.md @@ -3748,12 +3748,26 @@ Description -propagate
bool +disabled
bool + + +(Optional) +

+Disabled toggles the watermark propagation, defaults to false. +

+ + + + +maxDelay
+ +Kubernetes meta/v1.Duration (Optional)

-Propagate toggles the watermark propagation. +Maximum delay allowed for watermark calculation, defaults to “0s”, which +means no delay.

diff --git a/docs/assets/numa.svg b/docs/assets/numaproj.svg similarity index 100% rename from docs/assets/numa.svg rename to docs/assets/numaproj.svg diff --git a/docs/sources/http.md b/docs/sources/http.md index 966bac5955..a5debba1c1 100644 --- a/docs/sources/http.md +++ b/docs/sources/http.md @@ -79,6 +79,14 @@ When posting data to the HTTP Source, an optional HTTP header `x-numaflow-id` ca curl -kq -X POST -H "x-numaflow-id: ${id}" -d "hello world" ${http-source-url} ``` +## x-numaflow-event-time + +By default, the time of the date coming to the HTTP source is used as the event time, it could be set by putting an HTTP header `x-numaflow-event-time` with value of the number of seconds elapsed since January 1, 1970 UTC. + +```sh +curl -kq -X POST -H "x-numaflow-event-time: 1663006726" -d "hello world" ${http-source-url} +``` + ## Auth A `Bearer` token can be configured to prevent the HTTP Source from being accessed by unexpected clients. To do so, a Kubernetes Secret needs to be created to store the token, and the valid clients also need to include the token in its HTTP request header. diff --git a/go.mod b/go.mod index 345744fa37..177efbb71c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/numaproj/numaflow -go 1.18 +go 1.19 require ( github.com/Masterminds/sprig/v3 v3.2.2 diff --git a/mkdocs.yml b/mkdocs.yml index 80a26ece52..08d9d1fd12 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,7 +4,7 @@ edit_uri: edit/main/docs/ strict: true theme: name: material - favicon: assets/numa.svg + favicon: assets/numaproj.svg font: text: Roboto code: Roboto Mono diff --git a/pkg/apis/numaflow/v1alpha1/const.go b/pkg/apis/numaflow/v1alpha1/const.go index 19738ea90a..459afeea53 100644 --- a/pkg/apis/numaflow/v1alpha1/const.go +++ b/pkg/apis/numaflow/v1alpha1/const.go @@ -24,7 +24,8 @@ const ( KeyReplica = "numaflow.numaproj.io/replica" // ID key in the header of sources like http - KeyMetaID = "x-numaflow-id" + KeyMetaID = "x-numaflow-id" + KeyMetaEventTime = "x-numaflow-event-time" DefaultISBSvcName = "default" @@ -86,7 +87,8 @@ const ( EnvDebug = "NUMAFLOW_DEBUG" // Watermark - EnvWatermarkOn = "NUMAFLOW_WATERMARK_ON" + EnvWatermarkDisabled = "NUMAFLOW_WATERMARK_DISABLED" + EnvWatermarkMaxDelay = "NUMAFLOW_WATERMARK_MAX_DELAY" PathVarRun = "/var/run/numaflow" VertexMetricsPort = 2469 diff --git a/pkg/apis/numaflow/v1alpha1/generated.pb.go b/pkg/apis/numaflow/v1alpha1/generated.pb.go index 204e324665..6d2ead619b 100644 --- a/pkg/apis/numaflow/v1alpha1/generated.pb.go +++ b/pkg/apis/numaflow/v1alpha1/generated.pb.go @@ -1632,299 +1632,299 @@ func init() { } var fileDescriptor_9d0d1b17d3865563 = []byte{ - // 4665 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5c, 0x5d, 0x6c, 0x24, 0x57, - 0x56, 0x4e, 0xff, 0xd9, 0xdd, 0xa7, 0xed, 0xb1, 0x7d, 0x67, 0x32, 0xe9, 0x78, 0x13, 0xf7, 0xd0, - 0x51, 0xa2, 0x01, 0x76, 0xdb, 0xc4, 0x64, 0xd9, 0x59, 0xd8, 0x6c, 0xe2, 0xb6, 0xc7, 0x8e, 0x67, - 0xec, 0x89, 0x39, 0x6d, 0xcf, 0x90, 0x64, 0x45, 0xb8, 0xae, 0xbe, 0x6e, 0x57, 0xba, 0xba, 0xaa, - 0xb7, 0xea, 0xb6, 0x67, 0x1c, 0xb1, 0x02, 0x89, 0x87, 0x80, 0x00, 0xed, 0x4a, 0xbc, 0x20, 0xad, - 0x58, 0xf1, 0xb0, 0x12, 0x02, 0x69, 0x5f, 0x10, 0x48, 0x08, 0xb4, 0x12, 0x4f, 0x28, 0xbc, 0xa0, - 0x3c, 0x20, 0x11, 0xa4, 0x95, 0xb5, 0x31, 0x12, 0x6f, 0x88, 0x45, 0xfb, 0x36, 0x42, 0x02, 0xdd, - 0x9f, 0xfa, 0xed, 0x6a, 0x8f, 0xdd, 0x6d, 0x87, 0x07, 0xe6, 0xad, 0xeb, 0x9e, 0x73, 0xbe, 0x73, - 0xeb, 0xfe, 0x9c, 0x7b, 0x7e, 0x6e, 0x35, 0xac, 0xb7, 0x4d, 0x7e, 0xd0, 0xdf, 0xab, 0x1b, 0x4e, - 0x77, 0xd1, 0xee, 0x77, 0x69, 0xcf, 0x75, 0x3e, 0x90, 0x3f, 0xf6, 0x2d, 0xe7, 0xe1, 0x62, 0xaf, - 0xd3, 0x5e, 0xa4, 0x3d, 0xd3, 0x0b, 0x5b, 0x0e, 0x5f, 0xa5, 0x56, 0xef, 0x80, 0xbe, 0xba, 0xd8, - 0x66, 0x36, 0x73, 0x29, 0x67, 0xad, 0x7a, 0xcf, 0x75, 0xb8, 0x43, 0xbe, 0x12, 0x02, 0xd5, 0x7d, - 0xa0, 0xba, 0x2f, 0x56, 0xef, 0x75, 0xda, 0x75, 0x01, 0x14, 0xb6, 0xf8, 0x40, 0xf3, 0x5f, 0x8a, - 0xf4, 0xa0, 0xed, 0xb4, 0x9d, 0x45, 0x89, 0xb7, 0xd7, 0xdf, 0x97, 0x4f, 0xf2, 0x41, 0xfe, 0x52, - 0x7a, 0xe6, 0x6b, 0x9d, 0x5b, 0x5e, 0xdd, 0x74, 0x44, 0xb7, 0x16, 0x0d, 0xc7, 0x65, 0x8b, 0x87, - 0x03, 0x7d, 0x99, 0x7f, 0x2d, 0xe4, 0xe9, 0x52, 0xe3, 0xc0, 0xb4, 0x99, 0x7b, 0xe4, 0xbf, 0xcb, - 0xa2, 0xcb, 0x3c, 0xa7, 0xef, 0x1a, 0xec, 0x5c, 0x52, 0xde, 0x62, 0x97, 0x71, 0x9a, 0xa6, 0x6b, - 0x71, 0x98, 0x94, 0xdb, 0xb7, 0xb9, 0xd9, 0x1d, 0x54, 0xf3, 0x4b, 0x4f, 0x12, 0xf0, 0x8c, 0x03, - 0xd6, 0xa5, 0x49, 0xb9, 0xda, 0x7f, 0x4e, 0xc1, 0x95, 0xe5, 0x3d, 0x8f, 0xbb, 0xd4, 0xe0, 0xf7, - 0x99, 0xcb, 0xd9, 0x23, 0x72, 0x03, 0xf2, 0x36, 0xed, 0xb2, 0x4a, 0xe6, 0x46, 0xe6, 0x66, 0xa9, - 0x31, 0xf5, 0xf1, 0x71, 0xf5, 0x99, 0x93, 0xe3, 0x6a, 0xfe, 0x1e, 0xed, 0x32, 0x94, 0x14, 0x62, - 0xc0, 0x84, 0x7a, 0xdb, 0x4a, 0xee, 0x46, 0xe6, 0x66, 0x79, 0xe9, 0x8d, 0xfa, 0x88, 0xd3, 0x54, - 0x6f, 0x4a, 0x98, 0x06, 0x9c, 0x1c, 0x57, 0x27, 0xd4, 0x6f, 0xd4, 0xd0, 0xe4, 0x3d, 0xc8, 0x7b, - 0xa6, 0xdd, 0xa9, 0xe4, 0xa5, 0x8a, 0xd7, 0x47, 0x57, 0x61, 0xda, 0x9d, 0x46, 0x51, 0xbc, 0x81, - 0xf8, 0x85, 0x12, 0x94, 0x7c, 0x3b, 0x03, 0x73, 0x86, 0x63, 0x73, 0x2a, 0x06, 0x6a, 0x87, 0x75, - 0x7b, 0x16, 0xe5, 0xac, 0x52, 0x90, 0xaa, 0xee, 0x8c, 0xac, 0x6a, 0x25, 0x89, 0xd8, 0x78, 0xf6, - 0xe4, 0xb8, 0x3a, 0x37, 0xd0, 0x8c, 0x83, 0xba, 0xc9, 0x03, 0xc8, 0xf5, 0x5b, 0xfb, 0x95, 0x09, - 0xd9, 0x85, 0xaf, 0x8d, 0xdc, 0x85, 0xdd, 0xd5, 0xb5, 0xc6, 0xe4, 0xc9, 0x71, 0x35, 0xb7, 0xbb, - 0xba, 0x86, 0x02, 0x91, 0x74, 0xa0, 0x28, 0x56, 0x59, 0x8b, 0x72, 0x5a, 0x99, 0x94, 0xe8, 0xcb, - 0x23, 0xa3, 0x6f, 0x69, 0xa0, 0xc6, 0xd4, 0xc9, 0x71, 0xb5, 0xe8, 0x3f, 0x61, 0xa0, 0x80, 0xfc, - 0x51, 0x06, 0xa6, 0x6c, 0xa7, 0xc5, 0x9a, 0xcc, 0x62, 0x06, 0x77, 0xdc, 0x4a, 0xf1, 0x46, 0xee, - 0x66, 0x79, 0xe9, 0x9d, 0x91, 0x35, 0xc6, 0xd7, 0x66, 0xfd, 0x5e, 0x04, 0xfb, 0xb6, 0xcd, 0xdd, - 0xa3, 0xc6, 0x35, 0xbd, 0x3e, 0xa7, 0xa2, 0x24, 0x8c, 0x75, 0x82, 0xec, 0x42, 0x99, 0x3b, 0x96, - 0x58, 0xf7, 0xa6, 0x63, 0x7b, 0x95, 0x92, 0xec, 0xd3, 0x42, 0x5d, 0x6d, 0x19, 0xa1, 0xb9, 0x2e, - 0xf6, 0x7c, 0xfd, 0xf0, 0xd5, 0xfa, 0x4e, 0xc0, 0xd6, 0xb8, 0xaa, 0x81, 0xcb, 0x61, 0x9b, 0x87, - 0x51, 0x1c, 0xc2, 0x60, 0xc6, 0x63, 0x46, 0xdf, 0x35, 0xf9, 0x91, 0x98, 0x62, 0xf6, 0x88, 0x57, - 0x40, 0x0e, 0xf0, 0x2b, 0x69, 0xd0, 0xdb, 0x4e, 0xab, 0x19, 0xe7, 0x6e, 0x5c, 0x3d, 0x39, 0xae, - 0xce, 0x24, 0x1a, 0x31, 0x89, 0x49, 0x6c, 0x98, 0x35, 0xbb, 0xb4, 0xcd, 0xb6, 0xfb, 0x96, 0xd5, - 0x64, 0x86, 0xcb, 0xb8, 0x57, 0x29, 0xcb, 0x57, 0xb8, 0x99, 0xa6, 0x67, 0xd3, 0x31, 0xa8, 0xf5, - 0xf6, 0xde, 0x07, 0xcc, 0xe0, 0xc8, 0xf6, 0x99, 0xcb, 0x6c, 0x83, 0x35, 0x2a, 0xfa, 0x65, 0x66, - 0x37, 0x12, 0x48, 0x38, 0x80, 0x4d, 0xd6, 0x61, 0xae, 0xe7, 0x9a, 0x8e, 0xec, 0x82, 0x45, 0x3d, - 0x4f, 0x6c, 0xfc, 0xca, 0x94, 0x34, 0x06, 0xcf, 0x6b, 0x98, 0xb9, 0xed, 0x24, 0x03, 0x0e, 0xca, - 0x90, 0x9b, 0x50, 0xf4, 0x1b, 0x2b, 0xd3, 0x37, 0x32, 0x37, 0x0b, 0x6a, 0xd9, 0xf8, 0xb2, 0x18, - 0x50, 0xc9, 0x1a, 0x14, 0xe9, 0xfe, 0xbe, 0x69, 0x0b, 0xce, 0x2b, 0x72, 0x08, 0x5f, 0x48, 0x7b, - 0xb5, 0x65, 0xcd, 0xa3, 0x70, 0xfc, 0x27, 0x0c, 0x64, 0xc9, 0x1d, 0x20, 0x1e, 0x73, 0x0f, 0x4d, - 0x83, 0x2d, 0x1b, 0x86, 0xd3, 0xb7, 0xb9, 0xec, 0xfb, 0x8c, 0xec, 0xfb, 0xbc, 0xee, 0x3b, 0x69, - 0x0e, 0x70, 0x60, 0x8a, 0x14, 0xb9, 0x0d, 0x93, 0x87, 0x8e, 0xd5, 0xef, 0x32, 0xaf, 0x32, 0x2b, - 0x47, 0x7b, 0x3e, 0xad, 0x4b, 0xf7, 0x25, 0x4b, 0x63, 0x46, 0x83, 0x4f, 0xaa, 0x67, 0x0f, 0x7d, - 0x59, 0x62, 0xc2, 0x84, 0x65, 0x76, 0x4d, 0xee, 0x55, 0xe6, 0xe4, 0x8b, 0xdd, 0x1e, 0x79, 0x2b, - 0xa8, 0x2d, 0xb0, 0x29, 0xc1, 0x94, 0xc5, 0x54, 0xbf, 0x51, 0x2b, 0x20, 0x06, 0x14, 0x3c, 0x83, - 0x5a, 0xac, 0x42, 0xa4, 0xa6, 0xaf, 0x8f, 0x6e, 0x32, 0x05, 0x4a, 0x63, 0x5a, 0xbf, 0x53, 0x41, - 0x3e, 0xa2, 0xc2, 0x9e, 0x7f, 0x03, 0xe6, 0x06, 0x36, 0x21, 0x99, 0x85, 0x5c, 0x87, 0x1d, 0xa9, - 0x13, 0x03, 0xc5, 0x4f, 0x72, 0x0d, 0x0a, 0x87, 0xd4, 0xea, 0xb3, 0x4a, 0x56, 0xb6, 0xa9, 0x87, - 0x5f, 0xce, 0xde, 0xca, 0xd4, 0x1e, 0xc0, 0xf4, 0x72, 0x9f, 0x1f, 0x38, 0xae, 0xf9, 0xa1, 0xdc, - 0x47, 0x64, 0x0d, 0x0a, 0xdc, 0xe9, 0x30, 0x5b, 0x8a, 0x97, 0x97, 0x5e, 0x4e, 0x1b, 0x66, 0xb5, - 0x36, 0xef, 0xb2, 0x23, 0x5f, 0x6f, 0xa3, 0x24, 0x7a, 0xb6, 0x23, 0xe4, 0x50, 0x89, 0xd7, 0xde, - 0x85, 0x89, 0x46, 0x7f, 0x7f, 0x9f, 0xb9, 0x67, 0x38, 0xc1, 0xea, 0x90, 0xe7, 0x47, 0x3d, 0xdd, - 0xbb, 0x60, 0x69, 0xe4, 0x77, 0x8e, 0x7a, 0xec, 0xf1, 0x71, 0x15, 0x14, 0x8e, 0x78, 0x42, 0xc9, - 0x57, 0xfb, 0x69, 0x06, 0xae, 0xaa, 0x46, 0xbd, 0x7a, 0x56, 0x1c, 0x7b, 0xdf, 0x6c, 0x13, 0x06, - 0x05, 0x97, 0xb5, 0x4c, 0x4f, 0xf7, 0x7d, 0x75, 0xe4, 0x21, 0x47, 0x81, 0xa2, 0x40, 0xd5, 0xab, - 0xc9, 0x06, 0x54, 0xe8, 0xa4, 0x0f, 0xa5, 0x0f, 0x18, 0xf7, 0xb8, 0xcb, 0x68, 0x57, 0xf6, 0xb9, - 0xbc, 0xf4, 0xd6, 0xc8, 0xaa, 0xee, 0x30, 0xde, 0x94, 0x48, 0x5a, 0xdd, 0xf4, 0xc9, 0x71, 0xb5, - 0x14, 0x34, 0x62, 0xa8, 0xa9, 0xf6, 0xef, 0x59, 0x28, 0x05, 0x87, 0x17, 0x79, 0x09, 0x0a, 0xd2, - 0x56, 0xe8, 0x61, 0x0d, 0x96, 0x87, 0x34, 0x29, 0xa8, 0x68, 0xe4, 0x65, 0x98, 0x34, 0x9c, 0x6e, - 0x97, 0xda, 0xad, 0x4a, 0xf6, 0x46, 0xee, 0x66, 0xa9, 0x51, 0x16, 0xbb, 0x62, 0x45, 0x35, 0xa1, - 0x4f, 0x23, 0x2f, 0x40, 0x9e, 0xba, 0x6d, 0xaf, 0x92, 0x93, 0x3c, 0xf2, 0x74, 0x5e, 0x76, 0xdb, - 0x1e, 0xca, 0x56, 0xf2, 0x55, 0xc8, 0x31, 0xfb, 0xb0, 0x92, 0x1f, 0xbe, 0xed, 0x6e, 0xdb, 0x87, - 0xf7, 0xa9, 0xdb, 0x28, 0xeb, 0x3e, 0xe4, 0x6e, 0xdb, 0x87, 0x28, 0x64, 0xc8, 0x3b, 0x30, 0xa5, - 0x76, 0xde, 0x96, 0xd8, 0xc8, 0x5e, 0xa5, 0x20, 0x31, 0xaa, 0xc3, 0xb7, 0xae, 0xe4, 0x0b, 0x4f, - 0x91, 0x48, 0xa3, 0x87, 0x31, 0x28, 0xf2, 0x0e, 0x94, 0x7c, 0x2f, 0xcf, 0xd3, 0xe7, 0x74, 0xaa, - 0x01, 0x46, 0xcd, 0x84, 0xec, 0x9b, 0x7d, 0xd3, 0x65, 0x5d, 0x66, 0x73, 0xaf, 0x31, 0xa7, 0x15, - 0x94, 0x7c, 0xaa, 0x87, 0x21, 0x5a, 0xed, 0xbf, 0xb2, 0x30, 0xe8, 0x25, 0xc4, 0x15, 0x66, 0x2e, - 0x52, 0x21, 0xd9, 0x83, 0x99, 0xc0, 0xee, 0x6f, 0x3b, 0x96, 0x69, 0x1c, 0xe9, 0xad, 0x70, 0x4b, - 0x8b, 0xcd, 0x6c, 0xc4, 0xc9, 0x8f, 0x8f, 0xab, 0x2f, 0x0e, 0xfa, 0xc8, 0xf5, 0x90, 0x01, 0x93, - 0x80, 0x42, 0x47, 0xf2, 0x78, 0x54, 0xee, 0xe2, 0x4b, 0x43, 0x76, 0xf8, 0x08, 0x67, 0xe3, 0xe8, - 0x2b, 0xa5, 0xf6, 0xe7, 0x59, 0xc8, 0xdf, 0x6e, 0xb5, 0x99, 0xb0, 0x16, 0xfb, 0xae, 0xd3, 0x4d, - 0x5a, 0x8b, 0x35, 0xd7, 0xe9, 0xa2, 0xa4, 0x90, 0x79, 0xc8, 0x72, 0x47, 0x0f, 0x10, 0x68, 0x7a, - 0x76, 0xc7, 0xc1, 0x2c, 0x77, 0xc8, 0x87, 0x00, 0x86, 0x63, 0xb7, 0x4c, 0xe5, 0x5a, 0xe4, 0xc6, - 0xf4, 0x20, 0xd7, 0x1c, 0xf7, 0x21, 0x75, 0x5b, 0x2b, 0x01, 0x62, 0xe3, 0xca, 0xc9, 0x71, 0x15, - 0xc2, 0x67, 0x8c, 0x68, 0x23, 0xed, 0xe0, 0x6c, 0x51, 0x4e, 0xf2, 0xca, 0xc8, 0x7a, 0xc5, 0x40, - 0x0c, 0x3f, 0x59, 0x6a, 0x7f, 0x98, 0x01, 0x08, 0x59, 0xc8, 0xeb, 0x30, 0xb3, 0x27, 0x8d, 0xe1, - 0x16, 0x7d, 0xb4, 0xc9, 0xec, 0x36, 0x3f, 0x90, 0x83, 0x97, 0x57, 0x93, 0xd6, 0x88, 0x93, 0x30, - 0xc9, 0x4b, 0xde, 0x84, 0x59, 0xd5, 0xb4, 0xeb, 0x51, 0x8d, 0x29, 0x07, 0x77, 0xba, 0x71, 0x4d, - 0xb8, 0x28, 0x8d, 0x04, 0x0d, 0x07, 0xb8, 0x6b, 0xaf, 0xc1, 0xdc, 0xc0, 0x48, 0x91, 0x2a, 0x14, - 0x3a, 0xec, 0x68, 0x43, 0x9c, 0x23, 0xc2, 0xa8, 0x48, 0x2b, 0x7a, 0x57, 0x34, 0xa0, 0x6a, 0xaf, - 0xfd, 0x77, 0x06, 0x8a, 0x6b, 0x7d, 0xdb, 0x90, 0xa7, 0xce, 0x93, 0xcf, 0x08, 0xdf, 0x46, 0x65, - 0x53, 0x6d, 0x54, 0x1f, 0x26, 0x3a, 0x0f, 0x03, 0x1b, 0x56, 0x5e, 0xda, 0x1a, 0x7d, 0xce, 0x75, - 0x97, 0xea, 0x77, 0x25, 0x9e, 0x72, 0x6b, 0xaf, 0xe8, 0x0e, 0x4d, 0xdc, 0x7d, 0x20, 0x95, 0x6a, - 0x65, 0xf3, 0x5f, 0x85, 0x72, 0x84, 0xed, 0x5c, 0x07, 0xef, 0x0f, 0x32, 0x30, 0xb3, 0xae, 0xc2, - 0x3f, 0xc7, 0x55, 0xc1, 0x16, 0x79, 0x1e, 0x72, 0x6e, 0xaf, 0x2f, 0xe5, 0x73, 0x2a, 0x6e, 0xc0, - 0xed, 0x5d, 0x14, 0x6d, 0xe4, 0xd7, 0xa0, 0xd8, 0xea, 0x2b, 0x57, 0x57, 0x1f, 0x39, 0xf5, 0xc8, - 0xfe, 0x0a, 0x82, 0xcc, 0xf0, 0xcd, 0x44, 0x10, 0x20, 0x76, 0xdc, 0xaa, 0x96, 0x52, 0x5e, 0x9a, - 0xff, 0x84, 0x01, 0x9a, 0x38, 0x23, 0xba, 0x5e, 0xbb, 0x69, 0x7e, 0xa8, 0xe2, 0xc7, 0x82, 0x3a, - 0x23, 0xb6, 0x54, 0x13, 0xfa, 0xb4, 0xda, 0xb7, 0xb3, 0x70, 0x7d, 0x9d, 0xf1, 0x55, 0xca, 0xba, - 0x8e, 0xbd, 0xca, 0x7a, 0x96, 0x73, 0x24, 0x4c, 0x1b, 0xb2, 0x6f, 0x92, 0x37, 0x01, 0x4c, 0x6f, - 0xaf, 0x79, 0x68, 0x88, 0x23, 0x5a, 0x4f, 0xe1, 0x0d, 0x3d, 0x62, 0xb0, 0xd1, 0x6c, 0x68, 0xca, - 0xe3, 0xd8, 0x13, 0x46, 0x64, 0xc2, 0xc3, 0x2c, 0x7b, 0xca, 0x61, 0xd6, 0x04, 0xe8, 0x85, 0x06, - 0x32, 0x27, 0x39, 0x7f, 0xd1, 0x57, 0x73, 0x1e, 0xdb, 0x18, 0x81, 0x19, 0xc7, 0x64, 0xfd, 0x6d, - 0x0e, 0xe6, 0xd7, 0x19, 0x0f, 0xce, 0x6a, 0xed, 0x8b, 0x34, 0x7b, 0xcc, 0x10, 0xa3, 0xf2, 0x51, - 0x06, 0x26, 0x2c, 0xba, 0xc7, 0x2c, 0x4f, 0x6e, 0x81, 0xf2, 0xd2, 0xfb, 0x23, 0xaf, 0xc9, 0xe1, - 0x5a, 0xea, 0x9b, 0x52, 0x43, 0x62, 0x95, 0xaa, 0x46, 0xd4, 0xea, 0xc9, 0x97, 0xa1, 0x6c, 0x58, - 0x7d, 0x8f, 0x33, 0x77, 0xdb, 0x71, 0xd5, 0xe6, 0x2e, 0x84, 0x01, 0xd5, 0x4a, 0x48, 0xc2, 0x28, - 0x1f, 0x59, 0x02, 0x30, 0x2c, 0x93, 0xd9, 0x5c, 0x4a, 0xa9, 0xb5, 0x41, 0xfc, 0xf1, 0x5e, 0x09, - 0x28, 0x18, 0xe1, 0x12, 0xaa, 0xba, 0x8e, 0x6d, 0x72, 0x47, 0xa9, 0xca, 0xc7, 0x55, 0x6d, 0x85, - 0x24, 0x8c, 0xf2, 0x49, 0x31, 0xc6, 0x5d, 0xd3, 0xf0, 0xa4, 0x58, 0x21, 0x21, 0x16, 0x92, 0x30, - 0xca, 0x27, 0xb6, 0x5f, 0xe4, 0xfd, 0xcf, 0xb5, 0xfd, 0xfe, 0xae, 0x08, 0x0b, 0xb1, 0x61, 0xe5, - 0x94, 0xb3, 0xfd, 0xbe, 0xd5, 0x64, 0xdc, 0x9f, 0xc0, 0x2f, 0x43, 0x59, 0x07, 0x22, 0xf7, 0x42, - 0xd3, 0x14, 0x74, 0xaa, 0x19, 0x92, 0x30, 0xca, 0x47, 0x7e, 0x3f, 0x9c, 0xf7, 0xac, 0x9c, 0x77, - 0xe3, 0x62, 0xe6, 0x7d, 0xa0, 0x83, 0x67, 0x9a, 0xfb, 0x45, 0x28, 0xd9, 0x94, 0x7b, 0x72, 0x23, - 0xe9, 0x3d, 0x13, 0xf8, 0x22, 0xf7, 0x7c, 0x02, 0x86, 0x3c, 0x64, 0x1b, 0xae, 0xe9, 0x21, 0xbe, - 0xfd, 0xa8, 0xe7, 0xb8, 0x9c, 0xb9, 0x4a, 0x36, 0x2f, 0x65, 0x5f, 0xd0, 0xb2, 0xd7, 0xb6, 0x52, - 0x78, 0x30, 0x55, 0x92, 0x6c, 0xc1, 0x55, 0x43, 0xfa, 0xb6, 0xc8, 0x2c, 0x87, 0xb6, 0x7c, 0xc0, - 0x82, 0x04, 0xfc, 0x82, 0x06, 0xbc, 0xba, 0x32, 0xc8, 0x82, 0x69, 0x72, 0xc9, 0xd5, 0x3c, 0x31, - 0xd2, 0x6a, 0x9e, 0x1c, 0x65, 0x35, 0x17, 0x47, 0x5b, 0xcd, 0xa5, 0xb3, 0xad, 0x66, 0x31, 0xf2, - 0x62, 0x1d, 0x31, 0x57, 0x04, 0x64, 0x2a, 0xc4, 0x92, 0x0b, 0x0f, 0xe2, 0x23, 0xdf, 0x4c, 0xe1, - 0xc1, 0x54, 0x49, 0xb2, 0x07, 0xf3, 0xaa, 0xfd, 0xb6, 0x6d, 0xb8, 0x47, 0x3d, 0x61, 0xee, 0x23, - 0xb8, 0x65, 0x89, 0x5b, 0xd3, 0xb8, 0xf3, 0xcd, 0xa1, 0x9c, 0x78, 0x0a, 0x0a, 0xf9, 0x15, 0x98, - 0x56, 0xb3, 0xb4, 0x45, 0x7b, 0x91, 0xdc, 0xc4, 0xb3, 0x1a, 0x76, 0x7a, 0x25, 0x4a, 0xc4, 0x38, - 0x2f, 0x59, 0x86, 0x99, 0xde, 0xa1, 0x21, 0x7e, 0x6e, 0xec, 0xdf, 0x63, 0xac, 0xc5, 0x5a, 0x32, - 0x35, 0x51, 0x6a, 0x3c, 0xe7, 0x3b, 0xbe, 0xdb, 0x71, 0x32, 0x26, 0xf9, 0xc9, 0x2d, 0x98, 0xf2, - 0x38, 0x75, 0xb9, 0x0e, 0x6a, 0x64, 0xc2, 0xa2, 0x14, 0x46, 0x10, 0xcd, 0x08, 0x0d, 0x63, 0x9c, - 0xe3, 0x58, 0x8f, 0xc7, 0xea, 0x30, 0x94, 0x51, 0x61, 0xc2, 0xec, 0xff, 0x4e, 0xd2, 0xec, 0xbf, - 0x37, 0xce, 0xf6, 0x4f, 0xd1, 0x70, 0xa6, 0x6d, 0x7f, 0x07, 0x88, 0xab, 0x63, 0x58, 0x15, 0xc6, - 0x44, 0x2c, 0x7f, 0x90, 0x7a, 0xc1, 0x01, 0x0e, 0x4c, 0x91, 0x22, 0x4d, 0x78, 0xd6, 0x63, 0x36, - 0x37, 0x6d, 0x66, 0xc5, 0xe1, 0xd4, 0x91, 0xf0, 0xa2, 0x86, 0x7b, 0xb6, 0x99, 0xc6, 0x84, 0xe9, - 0xb2, 0xe3, 0x0c, 0xfe, 0x8f, 0x4a, 0xf2, 0xdc, 0x55, 0x43, 0x73, 0x61, 0x66, 0xfb, 0xa3, 0xa4, - 0xd9, 0x7e, 0x7f, 0xfc, 0x79, 0x1b, 0xcd, 0x64, 0x2f, 0x01, 0xc8, 0x59, 0x88, 0xda, 0xec, 0xc0, - 0x52, 0x61, 0x40, 0xc1, 0x08, 0x97, 0xd8, 0x85, 0xfe, 0x38, 0x47, 0xcd, 0x75, 0xb0, 0x0b, 0x9b, - 0x51, 0x22, 0xc6, 0x79, 0x87, 0x9a, 0xfc, 0xc2, 0xc8, 0x26, 0xff, 0x0e, 0x10, 0xd3, 0x36, 0x79, - 0x30, 0xe5, 0x0a, 0x6f, 0x22, 0x9e, 0xf9, 0xdb, 0x18, 0xe0, 0xc0, 0x14, 0xa9, 0x21, 0x4b, 0x79, - 0xf2, 0x62, 0x97, 0x72, 0x71, 0xf4, 0xa5, 0x4c, 0xde, 0x87, 0xe7, 0xa5, 0x2a, 0x3d, 0x3e, 0x71, - 0x60, 0x65, 0xfc, 0x7f, 0x46, 0x03, 0x3f, 0x8f, 0xc3, 0x18, 0x71, 0x38, 0x86, 0x98, 0x1f, 0xc3, - 0x65, 0x2d, 0xa1, 0x9c, 0x5a, 0xc3, 0x0f, 0x86, 0x95, 0x14, 0x1e, 0x4c, 0x95, 0x14, 0x4b, 0x8c, - 0x8b, 0x65, 0x48, 0xf7, 0x2c, 0xd6, 0x92, 0x07, 0x41, 0x31, 0x5c, 0x62, 0x3b, 0x9b, 0x4d, 0x4d, - 0xc1, 0x08, 0x57, 0x9a, 0xad, 0x9e, 0x3a, 0xa7, 0xad, 0x5e, 0x97, 0x65, 0x9e, 0xfd, 0xd8, 0x91, - 0xa0, 0x0d, 0x7e, 0x90, 0xcb, 0x5e, 0x49, 0x32, 0xe0, 0xa0, 0x8c, 0x3c, 0x2a, 0x0d, 0xd7, 0xec, - 0x71, 0x2f, 0x8e, 0x75, 0x25, 0x71, 0x54, 0xa6, 0xf0, 0x60, 0xaa, 0xa4, 0x70, 0x52, 0x0e, 0x18, - 0xb5, 0xf8, 0x41, 0x1c, 0x70, 0x26, 0xee, 0xa4, 0xbc, 0x35, 0xc8, 0x82, 0x69, 0x72, 0xe3, 0x98, - 0xb7, 0x3f, 0xc8, 0xc2, 0xd5, 0x75, 0xa6, 0x4b, 0x2c, 0xdb, 0x4e, 0xcb, 0xb7, 0x6b, 0xff, 0x4f, - 0xa3, 0xac, 0x3f, 0xc9, 0x00, 0xbc, 0xb5, 0xb3, 0xb3, 0xad, 0x43, 0xe4, 0x16, 0xe4, 0x69, 0x5f, - 0x67, 0x38, 0xca, 0x4b, 0x6b, 0xa3, 0x57, 0xb2, 0xa2, 0x49, 0x6f, 0x9d, 0x4e, 0xe8, 0xf3, 0x03, - 0x94, 0xe8, 0xe4, 0x67, 0x61, 0x52, 0x9f, 0x0d, 0x72, 0xac, 0x8a, 0x61, 0x45, 0x41, 0x9f, 0x1f, - 0xe8, 0xd3, 0x6b, 0x3f, 0xc9, 0xc2, 0xf5, 0x0d, 0x9b, 0x33, 0xb7, 0xc9, 0x59, 0x2f, 0x96, 0x94, - 0x26, 0xbf, 0x11, 0xa9, 0xf5, 0xa9, 0xfe, 0xfe, 0xc2, 0xd9, 0x62, 0x76, 0x55, 0x2f, 0xda, 0x62, - 0x9c, 0x86, 0xbb, 0x32, 0x6c, 0x8b, 0x14, 0xf8, 0xfa, 0x90, 0xf7, 0x7a, 0xcc, 0xd0, 0x19, 0x81, - 0xe6, 0xc8, 0xa3, 0x91, 0xfe, 0x02, 0x62, 0xe5, 0x85, 0xb9, 0x18, 0xb9, 0x0e, 0xa5, 0x3a, 0xf2, - 0x2d, 0x98, 0xf0, 0x38, 0xe5, 0x7d, 0x3f, 0xc3, 0xb6, 0x7b, 0xd1, 0x8a, 0x25, 0x78, 0x78, 0x40, - 0xaa, 0x67, 0xd4, 0x4a, 0x6b, 0x3f, 0xc9, 0xc0, 0x7c, 0xba, 0xe0, 0xa6, 0xe9, 0x71, 0xf2, 0x8d, - 0x81, 0x61, 0x3f, 0x63, 0xaa, 0x44, 0x48, 0xcb, 0x41, 0x9f, 0xd5, 0x8a, 0x8b, 0x7e, 0x4b, 0x64, - 0xc8, 0x39, 0x14, 0x4c, 0xce, 0xba, 0xbe, 0x97, 0xf0, 0xf6, 0x05, 0xbf, 0x7a, 0x64, 0x57, 0x0a, - 0x2d, 0xa8, 0x94, 0xd5, 0x3e, 0xca, 0x0e, 0x7b, 0x65, 0x31, 0x2d, 0xa4, 0x13, 0x2f, 0x7c, 0xdc, - 0x19, 0xaf, 0xf0, 0xd1, 0xe8, 0x47, 0xfa, 0x33, 0x58, 0xfe, 0xf8, 0xcd, 0xc1, 0xf2, 0xc7, 0xdb, - 0xe3, 0x97, 0x3f, 0x12, 0xa3, 0x30, 0xb4, 0x0a, 0xf2, 0xa3, 0x2c, 0xbc, 0x70, 0xda, 0xaa, 0x21, - 0xed, 0x60, 0x71, 0x66, 0xc6, 0xbd, 0x0e, 0x71, 0xea, 0x32, 0x24, 0x4b, 0x50, 0xe8, 0x1d, 0x50, - 0xcf, 0x37, 0xa7, 0xfe, 0xa9, 0x53, 0xd8, 0x16, 0x8d, 0x8f, 0x8f, 0xab, 0x65, 0x65, 0x86, 0xe5, - 0x23, 0x2a, 0x56, 0x61, 0x58, 0xba, 0xcc, 0xf3, 0x42, 0xc7, 0x2e, 0x30, 0x2c, 0x5b, 0xaa, 0x19, - 0x7d, 0x3a, 0xe1, 0x30, 0xa1, 0x82, 0x25, 0x9d, 0x4e, 0xde, 0x1c, 0xf9, 0x3d, 0x52, 0x4a, 0x65, - 0xe1, 0x4b, 0xe9, 0xb8, 0x5b, 0xeb, 0xaa, 0xfd, 0xe5, 0x15, 0xb8, 0x9e, 0x3e, 0x27, 0xa2, 0xef, - 0x87, 0xcc, 0xf5, 0x4c, 0xc7, 0xd6, 0xa7, 0x4f, 0x58, 0x66, 0x55, 0xcd, 0xe8, 0xd3, 0xc9, 0x4d, - 0x28, 0xba, 0xac, 0x67, 0x99, 0x06, 0xf5, 0x74, 0xd0, 0x21, 0xb3, 0x8f, 0xa8, 0xdb, 0x30, 0xa0, - 0x0e, 0xb9, 0xfa, 0x91, 0xfb, 0x3f, 0xbc, 0xfa, 0xf1, 0x67, 0x19, 0xe1, 0xcf, 0xa9, 0x8c, 0xc3, - 0x80, 0x80, 0x9e, 0x8b, 0x8b, 0xec, 0xd9, 0x8b, 0xca, 0x2f, 0x1c, 0xa2, 0x10, 0x87, 0xf7, 0x85, - 0x7c, 0x3f, 0x03, 0x95, 0x6e, 0xc2, 0x61, 0xbc, 0xc4, 0xdb, 0x33, 0x2f, 0x9c, 0x1c, 0x57, 0x2b, - 0x5b, 0x43, 0xf4, 0xe1, 0xd0, 0x9e, 0x90, 0xdf, 0x82, 0x72, 0x4f, 0xac, 0x0b, 0x8f, 0x33, 0xdb, - 0x60, 0xba, 0x56, 0x37, 0xfa, 0x6a, 0xde, 0x0e, 0xb1, 0x9a, 0xdc, 0xa5, 0x9c, 0xb5, 0x8f, 0x1a, - 0x33, 0x22, 0xb4, 0x8b, 0x10, 0x30, 0xaa, 0x31, 0x76, 0xe7, 0x66, 0xeb, 0xb2, 0xef, 0xdc, 0x7c, - 0x37, 0xfd, 0xce, 0x0d, 0xbd, 0x60, 0x0b, 0xf9, 0xf4, 0xee, 0xcd, 0xd3, 0xbb, 0x37, 0x9f, 0xd7, - 0xdd, 0x9b, 0x9b, 0x50, 0xf4, 0x18, 0xe7, 0xa6, 0xdd, 0xf6, 0x2a, 0xb3, 0xaa, 0x40, 0x27, 0xb4, - 0x36, 0x75, 0x1b, 0x06, 0x54, 0xf2, 0xf3, 0x50, 0x92, 0x29, 0xb6, 0x65, 0xb7, 0xed, 0x55, 0xe6, - 0x64, 0xa5, 0x4e, 0x9e, 0xe4, 0x4d, 0xbf, 0x11, 0x43, 0x3a, 0x79, 0x0d, 0xa6, 0x54, 0x29, 0x51, - 0x1d, 0x41, 0xf2, 0x9e, 0x4c, 0xa9, 0x31, 0x2b, 0x56, 0x70, 0x23, 0xd2, 0x8e, 0x31, 0x2e, 0x11, - 0xba, 0xb2, 0x20, 0x0f, 0x59, 0xb9, 0x1a, 0x0f, 0x5d, 0xc3, 0x0c, 0x25, 0x46, 0xb8, 0xc8, 0x8b, - 0x90, 0xe3, 0x96, 0x57, 0xb9, 0x26, 0x99, 0x83, 0x10, 0x63, 0x67, 0xb3, 0x89, 0xa2, 0x7d, 0xfc, - 0x4b, 0x34, 0xff, 0x93, 0x81, 0x99, 0xc4, 0x3d, 0x0e, 0xa1, 0xb3, 0xef, 0x5a, 0xfa, 0xa4, 0x0c, - 0x74, 0xee, 0xe2, 0x26, 0x8a, 0x76, 0xf2, 0xbe, 0x8e, 0x63, 0xb2, 0x63, 0xda, 0xa3, 0x7b, 0xcb, - 0x3b, 0x4d, 0x11, 0xb8, 0x0c, 0x84, 0x30, 0xb7, 0x12, 0xa3, 0x9b, 0x8b, 0xe7, 0x45, 0x4f, 0x1f, - 0xe1, 0x48, 0x72, 0x20, 0x7f, 0x96, 0xe4, 0x40, 0xed, 0xd3, 0x0c, 0x94, 0xee, 0xd2, 0xfd, 0x0e, - 0x6d, 0x9a, 0x76, 0x87, 0xbc, 0x0c, 0x93, 0x7b, 0xae, 0xd3, 0x61, 0xae, 0xa7, 0xab, 0xbf, 0xb2, - 0xa4, 0xd8, 0x50, 0x4d, 0xe8, 0xd3, 0x44, 0x3c, 0xca, 0x9d, 0x9e, 0x69, 0x24, 0xe3, 0xd1, 0x1d, - 0xd1, 0x88, 0x8a, 0x46, 0x1e, 0xa8, 0xb9, 0xcb, 0x8d, 0x79, 0x13, 0x73, 0x67, 0xb3, 0xa9, 0x2a, - 0xaa, 0xfe, 0xac, 0x93, 0x57, 0x62, 0xfe, 0x55, 0x69, 0xa8, 0x47, 0xf4, 0xdd, 0x2c, 0x94, 0xd5, - 0xab, 0xa9, 0x08, 0xf4, 0x22, 0x5f, 0xee, 0x0d, 0x99, 0x3c, 0xf7, 0xfa, 0x5d, 0xe6, 0xae, 0xbb, - 0x4e, 0xbf, 0xa7, 0x67, 0x29, 0x9a, 0x0c, 0x09, 0x89, 0x41, 0x02, 0x3d, 0x6c, 0xf2, 0x47, 0x27, - 0x7f, 0x89, 0xa3, 0x53, 0x38, 0x75, 0x74, 0xfe, 0x2a, 0x03, 0xa5, 0x4d, 0x73, 0x9f, 0x19, 0x47, - 0x86, 0xc5, 0xc8, 0x37, 0xa0, 0xd2, 0x62, 0x16, 0xe3, 0x6c, 0xdd, 0xa5, 0x06, 0xdb, 0x66, 0xae, - 0x29, 0x4d, 0xbd, 0x63, 0xb7, 0x94, 0x37, 0x5e, 0x08, 0x32, 0x16, 0x95, 0xd5, 0x21, 0x7c, 0x38, - 0x14, 0x81, 0x6c, 0xc0, 0x54, 0x8b, 0x79, 0xa6, 0xcb, 0x5a, 0xdb, 0x11, 0xbf, 0xfb, 0x65, 0x7f, - 0x49, 0xaf, 0x46, 0x68, 0x8f, 0x8f, 0xab, 0xd3, 0xdb, 0x66, 0x8f, 0x59, 0xa6, 0xcd, 0x94, 0x03, - 0x1e, 0x13, 0xad, 0x15, 0x20, 0xb7, 0xe9, 0xb4, 0x6b, 0xbf, 0x9b, 0x83, 0xe0, 0x0c, 0x27, 0xbf, - 0x97, 0x81, 0x32, 0xb5, 0x6d, 0x87, 0xeb, 0xc3, 0x51, 0xa5, 0xef, 0x71, 0x6c, 0x57, 0xa1, 0xbe, - 0x1c, 0x82, 0xaa, 0x93, 0x3a, 0x38, 0x50, 0x23, 0x14, 0x8c, 0xea, 0x26, 0xfd, 0x44, 0x32, 0x7a, - 0x6b, 0xfc, 0x5e, 0x9c, 0x21, 0xf5, 0x3c, 0xff, 0x75, 0x98, 0x4d, 0x76, 0xf6, 0x3c, 0x86, 0x70, - 0x9c, 0xb4, 0xd7, 0x9f, 0x66, 0xa0, 0xe8, 0x1b, 0x33, 0xb2, 0x02, 0xf9, 0xbe, 0xc7, 0xdc, 0xf3, - 0xdd, 0x41, 0x94, 0x16, 0x70, 0xd7, 0x63, 0x2e, 0x4a, 0x61, 0xf2, 0x36, 0x14, 0x7b, 0xd4, 0xf3, - 0x1e, 0x3a, 0x6e, 0x4b, 0x9b, 0xd9, 0x33, 0x02, 0xa9, 0xb3, 0x59, 0x8b, 0x62, 0x00, 0x52, 0xfb, - 0xe1, 0x34, 0x94, 0xef, 0x51, 0x6e, 0x1e, 0x32, 0x19, 0x0f, 0x5f, 0x4e, 0x40, 0xf4, 0xbd, 0x0c, - 0x5c, 0x8f, 0x67, 0xae, 0x2f, 0x31, 0x2a, 0x9a, 0x3f, 0x39, 0xae, 0x5e, 0xc7, 0x54, 0x6d, 0x38, - 0xa4, 0x17, 0x32, 0x3e, 0x1a, 0x48, 0x84, 0x5f, 0x76, 0x7c, 0xd4, 0x1c, 0xa6, 0x10, 0x87, 0xf7, - 0xe5, 0x69, 0x7c, 0x34, 0x42, 0x7c, 0x74, 0xe9, 0xdf, 0x24, 0x7c, 0x27, 0x3d, 0x3e, 0xba, 0x3f, - 0xba, 0x07, 0x14, 0xee, 0xc8, 0xa7, 0x41, 0xd1, 0xd3, 0xa0, 0xe8, 0xf3, 0x0a, 0x8a, 0x7a, 0x89, - 0xa0, 0x68, 0x9c, 0x62, 0x84, 0xae, 0xf2, 0x2b, 0xb4, 0x61, 0xc1, 0xd5, 0xf8, 0x61, 0xca, 0x1f, - 0x67, 0xe1, 0x6a, 0x8a, 0x75, 0x20, 0x6f, 0xc2, 0xac, 0xc7, 0x1d, 0x97, 0xb6, 0x59, 0x38, 0xa1, - 0xea, 0x40, 0x93, 0x37, 0x40, 0x9b, 0x09, 0x1a, 0x0e, 0x70, 0x93, 0xf7, 0x01, 0xa8, 0x61, 0x30, - 0xcf, 0xdb, 0x72, 0x5a, 0xbe, 0x5f, 0xf6, 0x86, 0x08, 0x17, 0x96, 0x83, 0xd6, 0xc7, 0xc7, 0xd5, - 0x2f, 0xa5, 0x15, 0x8c, 0xfc, 0xfe, 0x70, 0x75, 0xd7, 0x3b, 0x14, 0xc0, 0x08, 0x24, 0xf9, 0x75, - 0x00, 0x75, 0xfb, 0x3b, 0xb8, 0xa7, 0xf8, 0x84, 0xac, 0x7e, 0xdd, 0xbf, 0x5d, 0x5d, 0xff, 0xd5, - 0x3e, 0xb5, 0xb9, 0x58, 0x15, 0xf2, 0xee, 0xee, 0xfd, 0x00, 0x05, 0x23, 0x88, 0xb5, 0x7f, 0xc8, - 0x42, 0xd1, 0xf7, 0x17, 0x3f, 0x87, 0xba, 0x4d, 0x3b, 0x56, 0xb7, 0x19, 0xfd, 0x23, 0x14, 0xbf, - 0xcb, 0x43, 0x2b, 0x35, 0x4e, 0xa2, 0x52, 0xb3, 0x3e, 0xbe, 0xaa, 0xd3, 0x6b, 0x33, 0x3f, 0xc8, - 0xc2, 0x15, 0x9f, 0x55, 0xdf, 0x4f, 0xfe, 0x0a, 0x4c, 0xbb, 0x8c, 0xb6, 0x1a, 0x94, 0x1b, 0x07, - 0x72, 0xfa, 0xd4, 0xed, 0xe4, 0xb9, 0x93, 0xe3, 0xea, 0x34, 0x46, 0x09, 0x18, 0xe7, 0x4b, 0xbb, - 0xd8, 0x9c, 0x1d, 0xf3, 0x62, 0x73, 0xee, 0x3c, 0x17, 0x9b, 0x09, 0x85, 0xb2, 0xe8, 0xd1, 0x8e, - 0xd9, 0x65, 0x4e, 0x9f, 0x6b, 0xdf, 0xe6, 0xbc, 0xf7, 0x6e, 0xe5, 0xd9, 0x8b, 0x21, 0x0c, 0x46, - 0x31, 0x6b, 0xff, 0x9c, 0x81, 0xa9, 0x70, 0xbc, 0x2e, 0xbd, 0x7a, 0xb5, 0x1f, 0xaf, 0x5e, 0x2d, - 0x8f, 0xbd, 0x1c, 0x86, 0xd4, 0xab, 0xbe, 0x5f, 0x08, 0x5f, 0x4b, 0x56, 0xa8, 0xf6, 0x60, 0xde, - 0x4c, 0xad, 0xda, 0x44, 0xac, 0x4d, 0x70, 0x15, 0x6d, 0x63, 0x28, 0x27, 0x9e, 0x82, 0x42, 0xfa, - 0x50, 0x3c, 0x64, 0x2e, 0x37, 0x0d, 0xe6, 0xbf, 0xdf, 0xfa, 0x05, 0x7d, 0xe9, 0x18, 0x8e, 0xe9, - 0x7d, 0xad, 0x00, 0x03, 0x55, 0x64, 0x0f, 0x0a, 0xac, 0xd5, 0x66, 0xfe, 0xd5, 0xf3, 0xd7, 0xc7, - 0xba, 0xf6, 0x1f, 0x8e, 0xa7, 0x78, 0xf2, 0x50, 0x41, 0x13, 0x0f, 0x4a, 0x96, 0x1f, 0x65, 0xeb, - 0x75, 0xd8, 0x18, 0x59, 0x4f, 0x10, 0xaf, 0x87, 0x57, 0x41, 0x83, 0x26, 0x0c, 0xf5, 0x90, 0x4e, - 0xf0, 0x41, 0x43, 0xe1, 0x82, 0x8c, 0xc7, 0x29, 0x9f, 0xcb, 0x79, 0x50, 0x7a, 0x48, 0x39, 0x73, - 0xbb, 0xd4, 0xed, 0x68, 0x1f, 0x78, 0xf4, 0x37, 0x7c, 0xe0, 0x23, 0x85, 0x6f, 0x18, 0x34, 0x61, - 0xa8, 0xa7, 0xf6, 0xc3, 0x88, 0xb5, 0xfa, 0xbc, 0xcb, 0x87, 0xaf, 0xc5, 0xcb, 0x87, 0x0b, 0xc9, - 0xf2, 0x61, 0x22, 0x7f, 0x71, 0xfe, 0x02, 0x22, 0x85, 0xb2, 0x45, 0x3d, 0xbe, 0xdb, 0x6b, 0x51, - 0xae, 0x13, 0x79, 0xe5, 0xa5, 0x9f, 0x3b, 0x9b, 0x31, 0x11, 0xe6, 0x29, 0x74, 0x71, 0x37, 0x43, - 0x18, 0x8c, 0x62, 0xd6, 0xfe, 0x23, 0x03, 0x73, 0x03, 0x25, 0x63, 0x72, 0x00, 0x13, 0xb6, 0x74, - 0xca, 0xc7, 0xfe, 0x0e, 0x2f, 0xe2, 0xdb, 0xab, 0x45, 0xa3, 0x1b, 0x34, 0x3e, 0xb1, 0xa1, 0xc8, - 0x1e, 0x71, 0xe6, 0xda, 0xd4, 0xd2, 0x67, 0xe9, 0xc5, 0x7c, 0xf3, 0x27, 0x5d, 0xb0, 0xdb, 0x1a, - 0x19, 0x03, 0x1d, 0xb5, 0x9f, 0x66, 0xa1, 0x1c, 0xe1, 0x7b, 0x52, 0x92, 0x57, 0x5e, 0x45, 0x54, - 0xd1, 0xe9, 0xae, 0x6b, 0xe9, 0x89, 0x8e, 0x5c, 0x45, 0xd4, 0x24, 0xdc, 0xc4, 0x28, 0x1f, 0x59, - 0x02, 0xe8, 0x52, 0x8f, 0x33, 0x57, 0xda, 0xc6, 0xc4, 0x05, 0xc0, 0xad, 0x80, 0x82, 0x11, 0x2e, - 0x72, 0x43, 0x67, 0x4c, 0xf2, 0xf1, 0x0f, 0x68, 0x86, 0xa4, 0x43, 0x0a, 0x17, 0x90, 0x0e, 0x21, - 0x6d, 0x98, 0xf5, 0x7b, 0xed, 0x53, 0xf5, 0xc6, 0x3d, 0x23, 0xb0, 0xf2, 0x2e, 0x13, 0x10, 0x38, - 0x00, 0x5a, 0xfb, 0xeb, 0x0c, 0x4c, 0xc7, 0x5c, 0x64, 0xf2, 0x52, 0xf4, 0xbe, 0x43, 0x24, 0xb9, - 0x1a, 0xbb, 0xa7, 0xf0, 0x0a, 0x4c, 0xa8, 0x01, 0xd2, 0x03, 0x1f, 0x6c, 0x44, 0x35, 0x84, 0xa8, - 0xa9, 0x62, 0x4b, 0xe9, 0xec, 0x4b, 0x72, 0x4b, 0xe9, 0xf4, 0x0c, 0xfa, 0x74, 0xf2, 0x45, 0xe1, - 0xf4, 0xab, 0xde, 0xe9, 0x91, 0x0e, 0x0e, 0x06, 0xff, 0x3d, 0x30, 0xe0, 0xa8, 0x7d, 0x2f, 0x0f, - 0xea, 0x6b, 0x5d, 0x21, 0xd7, 0x32, 0x3d, 0x95, 0x50, 0xcf, 0xc8, 0x84, 0x7a, 0x20, 0xb7, 0xaa, - 0xdb, 0x31, 0xe0, 0x20, 0xcf, 0x43, 0xae, 0x6b, 0xda, 0x3a, 0x4f, 0x24, 0xd3, 0xb2, 0x5b, 0xa6, - 0x8d, 0xa2, 0x4d, 0x92, 0xe8, 0x23, 0x7d, 0xf3, 0x56, 0x91, 0xe8, 0x23, 0x14, 0x6d, 0xc2, 0x5b, - 0xb2, 0x1c, 0xa7, 0xb3, 0x47, 0x8d, 0x8e, 0x9f, 0x72, 0xcd, 0x4b, 0x6f, 0x47, 0x7a, 0x4b, 0x9b, - 0x71, 0x12, 0x26, 0x79, 0x85, 0xb8, 0xe1, 0x38, 0x56, 0xcb, 0x79, 0x68, 0xfb, 0xe2, 0x85, 0x50, - 0x7c, 0x25, 0x4e, 0xc2, 0x24, 0x2f, 0xd9, 0x85, 0xe7, 0x3e, 0x64, 0xae, 0xa3, 0x47, 0xac, 0x69, - 0x31, 0xd6, 0xf3, 0x61, 0x26, 0x24, 0xcc, 0x17, 0x4e, 0x8e, 0xab, 0xcf, 0xbd, 0x9b, 0xce, 0x82, - 0xc3, 0x64, 0x05, 0x2c, 0xa7, 0x6e, 0x9b, 0xf1, 0x6d, 0xd7, 0x11, 0xc1, 0x80, 0x69, 0xb7, 0x7d, - 0xd8, 0xc9, 0x10, 0x76, 0x27, 0x9d, 0x05, 0x87, 0xc9, 0x92, 0x15, 0x98, 0x53, 0xa4, 0x88, 0x13, - 0x28, 0xef, 0x80, 0x4e, 0xab, 0x8b, 0x02, 0x3b, 0x49, 0x22, 0x0e, 0xf2, 0x0b, 0xff, 0xd2, 0xcf, - 0xda, 0x6d, 0x33, 0x57, 0x4e, 0xb4, 0xbc, 0xee, 0xa9, 0xfd, 0x4b, 0x4c, 0xd0, 0x70, 0x80, 0x5b, - 0x7e, 0xf4, 0x28, 0x0b, 0x26, 0x0f, 0x20, 0x67, 0x39, 0x6d, 0x6d, 0x2e, 0x47, 0x4f, 0xe3, 0x6f, - 0x3a, 0x6d, 0xb5, 0x28, 0x36, 0x9d, 0x36, 0x0a, 0x44, 0x62, 0x40, 0xa1, 0x43, 0xf7, 0x3b, 0x54, - 0x5b, 0xc7, 0xd1, 0x4f, 0xd4, 0xa0, 0xb8, 0xa3, 0xbf, 0xe4, 0x13, 0x8f, 0xa8, 0xb0, 0x89, 0x01, - 0x13, 0xfd, 0x96, 0xfc, 0x77, 0x90, 0x71, 0xff, 0x80, 0x64, 0x77, 0x55, 0xaa, 0x90, 0xa6, 0x5e, - 0xfd, 0x46, 0x0d, 0x5d, 0xfb, 0xc7, 0x2c, 0xe8, 0xff, 0x24, 0x21, 0x7d, 0x28, 0xb5, 0xfd, 0x2f, - 0xe7, 0xf4, 0x98, 0xbd, 0x35, 0xc6, 0x65, 0xed, 0xd8, 0x37, 0x78, 0xaa, 0x5e, 0x19, 0x34, 0x62, - 0xa8, 0x89, 0xb0, 0xf8, 0x58, 0xae, 0x8e, 0x39, 0x96, 0x4a, 0xdd, 0xe0, 0x68, 0x52, 0xc8, 0x1f, - 0x70, 0xde, 0xd3, 0x63, 0x39, 0xfa, 0x47, 0xa4, 0xe1, 0xa5, 0x49, 0x95, 0x19, 0x17, 0xcf, 0x28, - 0xa1, 0x6b, 0x5d, 0xd0, 0xce, 0x08, 0x31, 0x62, 0xdf, 0xcb, 0xaa, 0x8a, 0xc7, 0xe2, 0xd9, 0x5c, - 0x84, 0xe0, 0x5b, 0xcf, 0xc8, 0x67, 0x34, 0xa9, 0x1f, 0xc6, 0xd6, 0xfe, 0x35, 0x0b, 0xb9, 0x9d, - 0xcd, 0xa6, 0xba, 0x15, 0x2e, 0xd3, 0x57, 0xac, 0xd9, 0x31, 0x7b, 0xf7, 0x99, 0x6b, 0xee, 0x1f, - 0x69, 0x7b, 0x18, 0xb9, 0x15, 0x9e, 0xe4, 0xc0, 0x14, 0x29, 0xf2, 0x1e, 0x4c, 0x19, 0x74, 0x85, - 0xb9, 0x5c, 0x1d, 0x2b, 0xe7, 0x4b, 0xf0, 0xcb, 0x1a, 0xf3, 0xca, 0x72, 0x28, 0x8e, 0x31, 0x30, - 0xb2, 0x0b, 0x60, 0x84, 0xd0, 0xb9, 0xf3, 0x40, 0xab, 0x0f, 0x84, 0x43, 0xe0, 0x08, 0x10, 0x41, - 0x28, 0x75, 0x04, 0xab, 0x44, 0xcd, 0x9f, 0x07, 0x55, 0x2e, 0xca, 0xbb, 0xbe, 0x2c, 0x86, 0x30, - 0xb5, 0x1f, 0x67, 0x20, 0xb7, 0xbb, 0xba, 0x46, 0x1c, 0x28, 0x05, 0x57, 0x99, 0xf4, 0x9e, 0x68, - 0x8c, 0x9f, 0xdb, 0x56, 0x8a, 0x83, 0x47, 0x0c, 0x75, 0x90, 0x03, 0x98, 0xdc, 0xeb, 0x9b, 0x16, - 0x37, 0x6d, 0x99, 0xfc, 0x1b, 0x27, 0x96, 0xf4, 0x3f, 0xb9, 0xd5, 0x95, 0x54, 0x85, 0x8a, 0x3e, - 0x7c, 0xed, 0x5b, 0xa0, 0x6d, 0x81, 0x88, 0x11, 0x2e, 0xe3, 0x25, 0x83, 0x18, 0x21, 0xed, 0x45, - 0x6b, 0x7f, 0x9f, 0x85, 0x09, 0xfd, 0x5f, 0x4c, 0x97, 0x9f, 0x18, 0x62, 0xb1, 0xc4, 0xd0, 0xca, - 0x98, 0xff, 0x4e, 0x32, 0x34, 0x2d, 0xd4, 0x4d, 0xa4, 0x85, 0xc6, 0xfd, 0x1b, 0x94, 0x27, 0x24, - 0x85, 0xfe, 0x29, 0x03, 0x57, 0x14, 0xe3, 0x86, 0xed, 0x71, 0x6a, 0x1b, 0xf2, 0x4f, 0xab, 0x0e, - 0x65, 0xcb, 0xd8, 0x61, 0x96, 0x8e, 0xd0, 0xe5, 0x99, 0xa1, 0x7e, 0xa3, 0x86, 0x16, 0x6e, 0xd7, - 0x81, 0xe3, 0x71, 0xf9, 0x65, 0x79, 0x36, 0xee, 0xae, 0xbd, 0xa5, 0xdb, 0x31, 0xe0, 0x48, 0xfa, - 0x81, 0x85, 0xe1, 0x7e, 0x60, 0xed, 0x2f, 0x32, 0x30, 0x15, 0xfd, 0x03, 0x98, 0xd1, 0x73, 0x5c, - 0x89, 0x14, 0x53, 0xf6, 0x12, 0x52, 0x4c, 0x9f, 0x64, 0x00, 0xfc, 0xce, 0x5e, 0x7a, 0x82, 0xa9, - 0x15, 0x4f, 0x30, 0x8d, 0x3d, 0xad, 0xe9, 0xe9, 0xa5, 0xbf, 0xc9, 0xfb, 0xaf, 0x24, 0x93, 0x4b, - 0x1f, 0x65, 0xe0, 0x0a, 0x8d, 0x25, 0x6c, 0xf4, 0x9b, 0x5d, 0x58, 0xfe, 0xe7, 0xba, 0xee, 0x46, - 0xe2, 0xdf, 0xd9, 0x30, 0xa1, 0x96, 0xdc, 0x82, 0xa9, 0x9e, 0x0e, 0xdb, 0xef, 0x85, 0xab, 0x2e, - 0x28, 0x42, 0x6d, 0x47, 0x68, 0x18, 0xe3, 0x7c, 0x42, 0x82, 0x2c, 0x77, 0x21, 0x09, 0xb2, 0x68, - 0x15, 0x3a, 0x7f, 0x6a, 0x15, 0xda, 0x86, 0xd2, 0xbe, 0xeb, 0x74, 0x65, 0x0e, 0x4a, 0xff, 0x6b, - 0xcb, 0x98, 0x79, 0xad, 0xc0, 0xc8, 0xae, 0xf9, 0xb8, 0x18, 0xaa, 0x10, 0xa7, 0x09, 0x77, 0x94, - 0xb6, 0x89, 0x8b, 0xd0, 0x16, 0x6c, 0xdd, 0x1d, 0x85, 0x8a, 0x3e, 0x7c, 0xed, 0x5f, 0xb2, 0xfe, - 0xd6, 0x6d, 0x26, 0xae, 0x71, 0x67, 0x86, 0x5c, 0xe3, 0xd6, 0x5f, 0xe2, 0x44, 0xb3, 0x30, 0xaf, - 0xc0, 0x84, 0xcb, 0xa8, 0xe7, 0xd8, 0xfa, 0x9b, 0xb6, 0xc0, 0xf0, 0xa1, 0x6c, 0x45, 0x4d, 0x8d, - 0x66, 0x6b, 0xb2, 0x4f, 0xc8, 0xd6, 0x7c, 0x31, 0x32, 0x37, 0x2a, 0x4b, 0x1d, 0x6c, 0xb3, 0x94, - 0xf9, 0x91, 0x81, 0xa8, 0x2e, 0xa0, 0x16, 0x92, 0x81, 0xa8, 0x2e, 0x72, 0x06, 0x1c, 0xa4, 0x05, - 0x53, 0x16, 0xf5, 0xb8, 0x0c, 0x3a, 0x5a, 0xcb, 0x7c, 0x84, 0x54, 0x50, 0xb0, 0x82, 0x37, 0x23, - 0x38, 0x18, 0x43, 0xad, 0x7d, 0x0d, 0xc2, 0x24, 0x1b, 0x59, 0x84, 0x52, 0xcf, 0x75, 0x7a, 0xb4, - 0x4d, 0x39, 0xd3, 0x2e, 0x5e, 0xb0, 0x02, 0xb6, 0x7d, 0x02, 0x86, 0x3c, 0x8d, 0xfa, 0xc7, 0x9f, - 0x2d, 0x3c, 0xf3, 0xc9, 0x67, 0x0b, 0xcf, 0x7c, 0xfa, 0xd9, 0xc2, 0x33, 0xbf, 0x7d, 0xb2, 0x90, - 0xf9, 0xf8, 0x64, 0x21, 0xf3, 0xc9, 0xc9, 0x42, 0xe6, 0xd3, 0x93, 0x85, 0xcc, 0x8f, 0x4f, 0x16, - 0x32, 0xdf, 0xf9, 0xb7, 0x85, 0x67, 0xde, 0x2d, 0xfa, 0xd3, 0xfc, 0xbf, 0x01, 0x00, 0x00, 0xff, - 0xff, 0xe1, 0x26, 0x2a, 0x0c, 0xd5, 0x52, 0x00, 0x00, + // 4671 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7c, 0x5d, 0x6c, 0x24, 0x57, + 0x56, 0x7f, 0xaa, 0x3f, 0xec, 0xee, 0xd3, 0xf6, 0xd8, 0xbe, 0x33, 0x99, 0xf4, 0x78, 0x27, 0xee, + 0xf9, 0x77, 0x94, 0x68, 0xfe, 0xb0, 0xdb, 0x26, 0x26, 0xcb, 0xce, 0x02, 0xd9, 0xc4, 0x6d, 0x8f, + 0x3d, 0x9e, 0xb1, 0x27, 0xe6, 0xb4, 0x3d, 0xb3, 0x49, 0x56, 0x0c, 0xe5, 0xea, 0xdb, 0xed, 0x4a, + 0x57, 0x57, 0xf5, 0x56, 0xdd, 0xf6, 0x8c, 0x23, 0x56, 0x20, 0xf1, 0x10, 0x10, 0xa0, 0x5d, 0x69, + 0x5f, 0x90, 0x56, 0xac, 0x78, 0x58, 0x09, 0x81, 0xb4, 0x2f, 0x08, 0x24, 0x04, 0x5a, 0x89, 0x27, + 0x14, 0x5e, 0x50, 0x1e, 0x90, 0x08, 0xd2, 0xca, 0xda, 0x18, 0x89, 0x37, 0xc4, 0xa2, 0x7d, 0x1b, + 0x21, 0x81, 0xee, 0x47, 0x7d, 0x76, 0xb5, 0xc7, 0xee, 0xb6, 0xc3, 0x03, 0xf3, 0xd6, 0x75, 0xcf, + 0x39, 0xbf, 0x73, 0xeb, 0x7e, 0x9c, 0x7b, 0x3e, 0x6e, 0x35, 0xac, 0xb7, 0x4d, 0xb6, 0xdf, 0xdf, + 0xab, 0x19, 0x4e, 0x77, 0xd1, 0xee, 0x77, 0xf5, 0x9e, 0xeb, 0x7c, 0x20, 0x7e, 0xb4, 0x2c, 0xe7, + 0xf1, 0x62, 0xaf, 0xd3, 0x5e, 0xd4, 0x7b, 0xa6, 0x17, 0xb6, 0x1c, 0xbc, 0xae, 0x5b, 0xbd, 0x7d, + 0xfd, 0xf5, 0xc5, 0x36, 0xb5, 0xa9, 0xab, 0x33, 0xda, 0xac, 0xf5, 0x5c, 0x87, 0x39, 0xe4, 0x2b, + 0x21, 0x50, 0xcd, 0x07, 0xaa, 0xf9, 0x62, 0xb5, 0x5e, 0xa7, 0x5d, 0xe3, 0x40, 0x61, 0x8b, 0x0f, + 0x34, 0xff, 0xa5, 0x48, 0x0f, 0xda, 0x4e, 0xdb, 0x59, 0x14, 0x78, 0x7b, 0xfd, 0x96, 0x78, 0x12, + 0x0f, 0xe2, 0x97, 0xd4, 0x33, 0x5f, 0xed, 0xdc, 0xf2, 0x6a, 0xa6, 0xc3, 0xbb, 0xb5, 0x68, 0x38, + 0x2e, 0x5d, 0x3c, 0x18, 0xe8, 0xcb, 0xfc, 0x1b, 0x21, 0x4f, 0x57, 0x37, 0xf6, 0x4d, 0x9b, 0xba, + 0x87, 0xfe, 0xbb, 0x2c, 0xba, 0xd4, 0x73, 0xfa, 0xae, 0x41, 0xcf, 0x24, 0xe5, 0x2d, 0x76, 0x29, + 0xd3, 0xd3, 0x74, 0x2d, 0x0e, 0x93, 0x72, 0xfb, 0x36, 0x33, 0xbb, 0x83, 0x6a, 0x7e, 0xe9, 0x59, + 0x02, 0x9e, 0xb1, 0x4f, 0xbb, 0x7a, 0x52, 0xae, 0xfa, 0x1f, 0x53, 0x70, 0x69, 0x79, 0xcf, 0x63, + 0xae, 0x6e, 0xb0, 0x07, 0xd4, 0x65, 0xf4, 0x09, 0xb9, 0x01, 0x39, 0x5b, 0xef, 0xd2, 0xb2, 0x76, + 0x43, 0xbb, 0x59, 0xac, 0x4f, 0x7d, 0x7c, 0x54, 0x79, 0xe1, 0xf8, 0xa8, 0x92, 0xbb, 0xaf, 0x77, + 0x29, 0x0a, 0x0a, 0x31, 0x60, 0x42, 0xbe, 0x6d, 0x39, 0x7b, 0x43, 0xbb, 0x59, 0x5a, 0x7a, 0xab, + 0x36, 0xe2, 0x34, 0xd5, 0x1a, 0x02, 0xa6, 0x0e, 0xc7, 0x47, 0x95, 0x09, 0xf9, 0x1b, 0x15, 0x34, + 0x79, 0x1f, 0x72, 0x9e, 0x69, 0x77, 0xca, 0x39, 0xa1, 0xe2, 0xcd, 0xd1, 0x55, 0x98, 0x76, 0xa7, + 0x5e, 0xe0, 0x6f, 0xc0, 0x7f, 0xa1, 0x00, 0x25, 0xdf, 0xd6, 0x60, 0xce, 0x70, 0x6c, 0xa6, 0xf3, + 0x81, 0xda, 0xa1, 0xdd, 0x9e, 0xa5, 0x33, 0x5a, 0xce, 0x0b, 0x55, 0x77, 0x47, 0x56, 0xb5, 0x92, + 0x44, 0xac, 0xbf, 0x78, 0x7c, 0x54, 0x99, 0x1b, 0x68, 0xc6, 0x41, 0xdd, 0xe4, 0x21, 0x64, 0xfb, + 0xcd, 0x56, 0x79, 0x42, 0x74, 0xe1, 0x57, 0x47, 0xee, 0xc2, 0xee, 0xea, 0x5a, 0x7d, 0xf2, 0xf8, + 0xa8, 0x92, 0xdd, 0x5d, 0x5d, 0x43, 0x8e, 0x48, 0x3a, 0x50, 0xe0, 0xab, 0xac, 0xa9, 0x33, 0xbd, + 0x3c, 0x29, 0xd0, 0x97, 0x47, 0x46, 0xdf, 0x52, 0x40, 0xf5, 0xa9, 0xe3, 0xa3, 0x4a, 0xc1, 0x7f, + 0xc2, 0x40, 0x01, 0xf9, 0xae, 0x06, 0x53, 0xb6, 0xd3, 0xa4, 0x0d, 0x6a, 0x51, 0x83, 0x39, 0x6e, + 0xb9, 0x70, 0x23, 0x7b, 0xb3, 0xb4, 0xf4, 0xee, 0xc8, 0x1a, 0xe3, 0x6b, 0xb3, 0x76, 0x3f, 0x82, + 0x7d, 0xdb, 0x66, 0xee, 0x61, 0xfd, 0x8a, 0x5a, 0x9f, 0x53, 0x51, 0x12, 0xc6, 0x3a, 0x41, 0x76, + 0xa1, 0xc4, 0x1c, 0x8b, 0xaf, 0x7b, 0xd3, 0xb1, 0xbd, 0x72, 0x51, 0xf4, 0x69, 0xa1, 0x26, 0xb7, + 0x0c, 0xd7, 0x5c, 0xe3, 0x7b, 0xbe, 0x76, 0xf0, 0x7a, 0x6d, 0x27, 0x60, 0xab, 0x5f, 0x56, 0xc0, + 0xa5, 0xb0, 0xcd, 0xc3, 0x28, 0x0e, 0xa1, 0x30, 0xe3, 0x51, 0xa3, 0xef, 0x9a, 0xec, 0x90, 0x4f, + 0x31, 0x7d, 0xc2, 0xca, 0x20, 0x06, 0xf8, 0xb5, 0x34, 0xe8, 0x6d, 0xa7, 0xd9, 0x88, 0x73, 0xd7, + 0x2f, 0x1f, 0x1f, 0x55, 0x66, 0x12, 0x8d, 0x98, 0xc4, 0x24, 0x36, 0xcc, 0x9a, 0x5d, 0xbd, 0x4d, + 0xb7, 0xfb, 0x96, 0xd5, 0xa0, 0x86, 0x4b, 0x99, 0x57, 0x2e, 0x89, 0x57, 0xb8, 0x99, 0xa6, 0x67, + 0xd3, 0x31, 0x74, 0xeb, 0x9d, 0xbd, 0x0f, 0xa8, 0xc1, 0x90, 0xb6, 0xa8, 0x4b, 0x6d, 0x83, 0xd6, + 0xcb, 0xea, 0x65, 0x66, 0x37, 0x12, 0x48, 0x38, 0x80, 0x4d, 0xd6, 0x61, 0xae, 0xe7, 0x9a, 0x8e, + 0xe8, 0x82, 0xa5, 0x7b, 0x1e, 0xdf, 0xf8, 0xe5, 0x29, 0x61, 0x0c, 0xae, 0x29, 0x98, 0xb9, 0xed, + 0x24, 0x03, 0x0e, 0xca, 0x90, 0x9b, 0x50, 0xf0, 0x1b, 0xcb, 0xd3, 0x37, 0xb4, 0x9b, 0x79, 0xb9, + 0x6c, 0x7c, 0x59, 0x0c, 0xa8, 0x64, 0x0d, 0x0a, 0x7a, 0xab, 0x65, 0xda, 0x9c, 0xf3, 0x92, 0x18, + 0xc2, 0xeb, 0x69, 0xaf, 0xb6, 0xac, 0x78, 0x24, 0x8e, 0xff, 0x84, 0x81, 0x2c, 0xb9, 0x0b, 0xc4, + 0xa3, 0xee, 0x81, 0x69, 0xd0, 0x65, 0xc3, 0x70, 0xfa, 0x36, 0x13, 0x7d, 0x9f, 0x11, 0x7d, 0x9f, + 0x57, 0x7d, 0x27, 0x8d, 0x01, 0x0e, 0x4c, 0x91, 0x22, 0xb7, 0x61, 0xf2, 0xc0, 0xb1, 0xfa, 0x5d, + 0xea, 0x95, 0x67, 0xc5, 0x68, 0xcf, 0xa7, 0x75, 0xe9, 0x81, 0x60, 0xa9, 0xcf, 0x28, 0xf0, 0x49, + 0xf9, 0xec, 0xa1, 0x2f, 0x4b, 0x4c, 0x98, 0xb0, 0xcc, 0xae, 0xc9, 0xbc, 0xf2, 0x9c, 0x78, 0xb1, + 0xdb, 0x23, 0x6f, 0x05, 0xb9, 0x05, 0x36, 0x05, 0x98, 0xb4, 0x98, 0xf2, 0x37, 0x2a, 0x05, 0xc4, + 0x80, 0xbc, 0x67, 0xe8, 0x16, 0x2d, 0x13, 0xa1, 0xe9, 0x6b, 0xa3, 0x9b, 0x4c, 0x8e, 0x52, 0x9f, + 0x56, 0xef, 0x94, 0x17, 0x8f, 0x28, 0xb1, 0xe7, 0xdf, 0x82, 0xb9, 0x81, 0x4d, 0x48, 0x66, 0x21, + 0xdb, 0xa1, 0x87, 0xf2, 0xc4, 0x40, 0xfe, 0x93, 0x5c, 0x81, 0xfc, 0x81, 0x6e, 0xf5, 0x69, 0x39, + 0x23, 0xda, 0xe4, 0xc3, 0x2f, 0x67, 0x6e, 0x69, 0xd5, 0x87, 0x30, 0xbd, 0xdc, 0x67, 0xfb, 0x8e, + 0x6b, 0x7e, 0x28, 0xf6, 0x11, 0x59, 0x83, 0x3c, 0x73, 0x3a, 0xd4, 0x16, 0xe2, 0xa5, 0xa5, 0x57, + 0xd3, 0x86, 0x59, 0xae, 0xcd, 0x7b, 0xf4, 0xd0, 0xd7, 0x5b, 0x2f, 0xf2, 0x9e, 0xed, 0x70, 0x39, + 0x94, 0xe2, 0xd5, 0xf7, 0x60, 0xa2, 0xde, 0x6f, 0xb5, 0xa8, 0x7b, 0x8a, 0x13, 0xac, 0x06, 0x39, + 0x76, 0xd8, 0x53, 0xbd, 0x0b, 0x96, 0x46, 0x6e, 0xe7, 0xb0, 0x47, 0x9f, 0x1e, 0x55, 0x40, 0xe2, + 0xf0, 0x27, 0x14, 0x7c, 0xd5, 0x9f, 0x69, 0x70, 0x59, 0x36, 0xaa, 0xd5, 0xb3, 0xe2, 0xd8, 0x2d, + 0xb3, 0x4d, 0x28, 0xe4, 0x5d, 0xda, 0x34, 0x3d, 0xd5, 0xf7, 0xd5, 0x91, 0x87, 0x1c, 0x39, 0x8a, + 0x04, 0x95, 0xaf, 0x26, 0x1a, 0x50, 0xa2, 0x93, 0x3e, 0x14, 0x3f, 0xa0, 0xcc, 0x63, 0x2e, 0xd5, + 0xbb, 0xa2, 0xcf, 0xa5, 0xa5, 0x3b, 0x23, 0xab, 0xba, 0x4b, 0x59, 0x43, 0x20, 0x29, 0x75, 0xd3, + 0xc7, 0x47, 0x95, 0x62, 0xd0, 0x88, 0xa1, 0xa6, 0xea, 0xbf, 0x65, 0xa0, 0x18, 0x1c, 0x5e, 0xe4, + 0x15, 0xc8, 0x0b, 0x5b, 0xa1, 0x86, 0x35, 0x58, 0x1e, 0xc2, 0xa4, 0xa0, 0xa4, 0x91, 0x57, 0x61, + 0xd2, 0x70, 0xba, 0x5d, 0xdd, 0x6e, 0x96, 0x33, 0x37, 0xb2, 0x37, 0x8b, 0xf5, 0x12, 0xdf, 0x15, + 0x2b, 0xb2, 0x09, 0x7d, 0x1a, 0xb9, 0x0e, 0x39, 0xdd, 0x6d, 0x7b, 0xe5, 0xac, 0xe0, 0x11, 0xa7, + 0xf3, 0xb2, 0xdb, 0xf6, 0x50, 0xb4, 0x92, 0xaf, 0x42, 0x96, 0xda, 0x07, 0xe5, 0xdc, 0xf0, 0x6d, + 0x77, 0xdb, 0x3e, 0x78, 0xa0, 0xbb, 0xf5, 0x92, 0xea, 0x43, 0xf6, 0xb6, 0x7d, 0x80, 0x5c, 0x86, + 0xbc, 0x0b, 0x53, 0x72, 0xe7, 0x6d, 0xf1, 0x8d, 0xec, 0x95, 0xf3, 0x02, 0xa3, 0x32, 0x7c, 0xeb, + 0x0a, 0xbe, 0xf0, 0x14, 0x89, 0x34, 0x7a, 0x18, 0x83, 0x22, 0xef, 0x42, 0xd1, 0xf7, 0xf2, 0x3c, + 0x75, 0x4e, 0xa7, 0x1a, 0x60, 0x54, 0x4c, 0x48, 0xbf, 0xd9, 0x37, 0x5d, 0xda, 0xa5, 0x36, 0xf3, + 0xea, 0x73, 0x4a, 0x41, 0xd1, 0xa7, 0x7a, 0x18, 0xa2, 0x55, 0xff, 0x33, 0x03, 0x83, 0x5e, 0x42, + 0x5c, 0xa1, 0x76, 0x9e, 0x0a, 0xc9, 0x1e, 0xcc, 0x04, 0x76, 0x7f, 0xdb, 0xb1, 0x4c, 0xe3, 0x50, + 0x6d, 0x85, 0x5b, 0x4a, 0x6c, 0x66, 0x23, 0x4e, 0x7e, 0x7a, 0x54, 0x79, 0x79, 0xd0, 0x47, 0xae, + 0x85, 0x0c, 0x98, 0x04, 0xe4, 0x3a, 0x92, 0xc7, 0xa3, 0x74, 0x17, 0x5f, 0x19, 0xb2, 0xc3, 0x47, + 0x38, 0x1b, 0x47, 0x5f, 0x29, 0xd5, 0x3f, 0xcb, 0x40, 0xee, 0x76, 0xb3, 0x4d, 0xb9, 0xb5, 0x68, + 0xb9, 0x4e, 0x37, 0x69, 0x2d, 0xd6, 0x5c, 0xa7, 0x8b, 0x82, 0x42, 0xe6, 0x21, 0xc3, 0x1c, 0x35, + 0x40, 0xa0, 0xe8, 0x99, 0x1d, 0x07, 0x33, 0xcc, 0x21, 0x1f, 0x02, 0x18, 0x8e, 0xdd, 0x34, 0xa5, + 0x6b, 0x91, 0x1d, 0xd3, 0x83, 0x5c, 0x73, 0xdc, 0xc7, 0xba, 0xdb, 0x5c, 0x09, 0x10, 0xeb, 0x97, + 0x8e, 0x8f, 0x2a, 0x10, 0x3e, 0x63, 0x44, 0x1b, 0x69, 0x07, 0x67, 0x8b, 0x74, 0x92, 0x57, 0x46, + 0xd6, 0xcb, 0x07, 0x62, 0xf8, 0xc9, 0x52, 0xfd, 0x43, 0x0d, 0x20, 0x64, 0x21, 0x6f, 0xc2, 0xcc, + 0x9e, 0x30, 0x86, 0x5b, 0xfa, 0x93, 0x4d, 0x6a, 0xb7, 0xd9, 0xbe, 0x18, 0xbc, 0x9c, 0x9c, 0xb4, + 0x7a, 0x9c, 0x84, 0x49, 0x5e, 0xf2, 0x36, 0xcc, 0xca, 0xa6, 0x5d, 0x4f, 0x57, 0x98, 0x62, 0x70, + 0xa7, 0xeb, 0x57, 0xb8, 0x8b, 0x52, 0x4f, 0xd0, 0x70, 0x80, 0xbb, 0xfa, 0x06, 0xcc, 0x0d, 0x8c, + 0x14, 0xa9, 0x40, 0xbe, 0x43, 0x0f, 0x37, 0xf8, 0x39, 0xc2, 0x8d, 0x8a, 0xb0, 0xa2, 0xf7, 0x78, + 0x03, 0xca, 0xf6, 0xea, 0x7f, 0x69, 0x50, 0x58, 0xeb, 0xdb, 0x86, 0x38, 0x75, 0x9e, 0x7d, 0x46, + 0xf8, 0x36, 0x2a, 0x93, 0x6a, 0xa3, 0xfa, 0x30, 0xd1, 0x79, 0x1c, 0xd8, 0xb0, 0xd2, 0xd2, 0xd6, + 0xe8, 0x73, 0xae, 0xba, 0x54, 0xbb, 0x27, 0xf0, 0xa4, 0x5b, 0x7b, 0x49, 0x75, 0x68, 0xe2, 0xde, + 0x43, 0xa1, 0x54, 0x29, 0x9b, 0xff, 0x2a, 0x94, 0x22, 0x6c, 0x67, 0x3a, 0x78, 0x7f, 0xa8, 0xc1, + 0xcc, 0xba, 0x0c, 0xff, 0x1c, 0x57, 0x06, 0x5b, 0xe4, 0x1a, 0x64, 0xdd, 0x5e, 0x5f, 0xc8, 0x67, + 0x65, 0xdc, 0x80, 0xdb, 0xbb, 0xc8, 0xdb, 0xc8, 0xd7, 0xa1, 0xd0, 0xec, 0x4b, 0x57, 0x57, 0x1d, + 0x39, 0xb5, 0xc8, 0xfe, 0x0a, 0x82, 0xcc, 0xf0, 0xcd, 0x78, 0x10, 0xc0, 0x77, 0xdc, 0xaa, 0x92, + 0x92, 0x5e, 0x9a, 0xff, 0x84, 0x01, 0x1a, 0x3f, 0x23, 0xba, 0x5e, 0xbb, 0x61, 0x7e, 0x28, 0xe3, + 0xc7, 0xbc, 0x3c, 0x23, 0xb6, 0x64, 0x13, 0xfa, 0xb4, 0xea, 0xb7, 0x33, 0x70, 0x75, 0x9d, 0xb2, + 0x55, 0x9d, 0x76, 0x1d, 0x7b, 0x95, 0xf6, 0x2c, 0xe7, 0x90, 0x9b, 0x36, 0xa4, 0xdf, 0x24, 0x6f, + 0x03, 0x98, 0xde, 0x5e, 0xe3, 0xc0, 0xe0, 0x47, 0xb4, 0x9a, 0xc2, 0x1b, 0x6a, 0xc4, 0x60, 0xa3, + 0x51, 0x57, 0x94, 0xa7, 0xb1, 0x27, 0x8c, 0xc8, 0x84, 0x87, 0x59, 0xe6, 0x84, 0xc3, 0xac, 0x01, + 0xd0, 0x0b, 0x0d, 0x64, 0x56, 0x70, 0xfe, 0xa2, 0xaf, 0xe6, 0x2c, 0xb6, 0x31, 0x02, 0x33, 0x8e, + 0xc9, 0xfa, 0x9b, 0x2c, 0xcc, 0xaf, 0x53, 0x16, 0x9c, 0xd5, 0xca, 0x17, 0x69, 0xf4, 0xa8, 0xc1, + 0x47, 0xe5, 0x23, 0x0d, 0x26, 0x2c, 0x7d, 0x8f, 0x5a, 0x9e, 0xd8, 0x02, 0xa5, 0xa5, 0x47, 0x23, + 0xaf, 0xc9, 0xe1, 0x5a, 0x6a, 0x9b, 0x42, 0x43, 0x62, 0x95, 0xca, 0x46, 0x54, 0xea, 0xc9, 0x97, + 0xa1, 0x64, 0x58, 0x7d, 0x8f, 0x51, 0x77, 0xdb, 0x71, 0xe5, 0xe6, 0xce, 0x87, 0x01, 0xd5, 0x4a, + 0x48, 0xc2, 0x28, 0x1f, 0x59, 0x02, 0x30, 0x2c, 0x93, 0xda, 0x4c, 0x48, 0xc9, 0xb5, 0x41, 0xfc, + 0xf1, 0x5e, 0x09, 0x28, 0x18, 0xe1, 0xe2, 0xaa, 0xba, 0x8e, 0x6d, 0x32, 0x47, 0xaa, 0xca, 0xc5, + 0x55, 0x6d, 0x85, 0x24, 0x8c, 0xf2, 0x09, 0x31, 0xca, 0x5c, 0xd3, 0xf0, 0x84, 0x58, 0x3e, 0x21, + 0x16, 0x92, 0x30, 0xca, 0xc7, 0xb7, 0x5f, 0xe4, 0xfd, 0xcf, 0xb4, 0xfd, 0xfe, 0xb6, 0x00, 0x0b, + 0xb1, 0x61, 0x65, 0x3a, 0xa3, 0xad, 0xbe, 0xd5, 0xa0, 0xcc, 0x9f, 0xc0, 0x2f, 0x43, 0x49, 0x05, + 0x22, 0xf7, 0x43, 0xd3, 0x14, 0x74, 0xaa, 0x11, 0x92, 0x30, 0xca, 0x47, 0x7e, 0x3f, 0x9c, 0xf7, + 0x8c, 0x98, 0x77, 0xe3, 0x7c, 0xe6, 0x7d, 0xa0, 0x83, 0xa7, 0x9a, 0xfb, 0x45, 0x28, 0xda, 0x3a, + 0xf3, 0xc4, 0x46, 0x52, 0x7b, 0x26, 0xf0, 0x45, 0xee, 0xfb, 0x04, 0x0c, 0x79, 0xc8, 0x36, 0x5c, + 0x51, 0x43, 0x7c, 0xfb, 0x49, 0xcf, 0x71, 0x19, 0x75, 0xa5, 0x6c, 0x4e, 0xc8, 0x5e, 0x57, 0xb2, + 0x57, 0xb6, 0x52, 0x78, 0x30, 0x55, 0x92, 0x6c, 0xc1, 0x65, 0x43, 0xf8, 0xb6, 0x48, 0x2d, 0x47, + 0x6f, 0xfa, 0x80, 0x79, 0x01, 0xf8, 0x05, 0x05, 0x78, 0x79, 0x65, 0x90, 0x05, 0xd3, 0xe4, 0x92, + 0xab, 0x79, 0x62, 0xa4, 0xd5, 0x3c, 0x39, 0xca, 0x6a, 0x2e, 0x8c, 0xb6, 0x9a, 0x8b, 0xa7, 0x5b, + 0xcd, 0x7c, 0xe4, 0xf9, 0x3a, 0xa2, 0x2e, 0x0f, 0xc8, 0x64, 0x88, 0x25, 0x16, 0x1e, 0xc4, 0x47, + 0xbe, 0x91, 0xc2, 0x83, 0xa9, 0x92, 0x64, 0x0f, 0xe6, 0x65, 0xfb, 0x6d, 0xdb, 0x70, 0x0f, 0x7b, + 0xdc, 0xdc, 0x47, 0x70, 0x4b, 0x02, 0xb7, 0xaa, 0x70, 0xe7, 0x1b, 0x43, 0x39, 0xf1, 0x04, 0x14, + 0xf2, 0x2b, 0x30, 0x2d, 0x67, 0x69, 0x4b, 0xef, 0x45, 0x72, 0x13, 0x2f, 0x2a, 0xd8, 0xe9, 0x95, + 0x28, 0x11, 0xe3, 0xbc, 0x64, 0x19, 0x66, 0x7a, 0x07, 0x06, 0xff, 0xb9, 0xd1, 0xba, 0x4f, 0x69, + 0x93, 0x36, 0x45, 0x6a, 0xa2, 0x58, 0x7f, 0xc9, 0x77, 0x7c, 0xb7, 0xe3, 0x64, 0x4c, 0xf2, 0x93, + 0x5b, 0x30, 0xe5, 0x31, 0xdd, 0x65, 0x2a, 0xa8, 0x11, 0x09, 0x8b, 0x62, 0x18, 0x41, 0x34, 0x22, + 0x34, 0x8c, 0x71, 0x8e, 0x63, 0x3d, 0x9e, 0xca, 0xc3, 0x50, 0x44, 0x85, 0x09, 0xb3, 0xff, 0x3b, + 0x49, 0xb3, 0xff, 0xfe, 0x38, 0xdb, 0x3f, 0x45, 0xc3, 0xa9, 0xb6, 0xfd, 0x5d, 0x20, 0xae, 0x8a, + 0x61, 0x65, 0x18, 0x13, 0xb1, 0xfc, 0x41, 0xea, 0x05, 0x07, 0x38, 0x30, 0x45, 0x8a, 0x34, 0xe0, + 0x45, 0x8f, 0xda, 0xcc, 0xb4, 0xa9, 0x15, 0x87, 0x93, 0x47, 0xc2, 0xcb, 0x0a, 0xee, 0xc5, 0x46, + 0x1a, 0x13, 0xa6, 0xcb, 0x8e, 0x33, 0xf8, 0x3f, 0x2e, 0x8a, 0x73, 0x57, 0x0e, 0xcd, 0xb9, 0x99, + 0xed, 0x8f, 0x92, 0x66, 0xfb, 0xd1, 0xf8, 0xf3, 0x36, 0x9a, 0xc9, 0x5e, 0x02, 0x10, 0xb3, 0x10, + 0xb5, 0xd9, 0x81, 0xa5, 0xc2, 0x80, 0x82, 0x11, 0x2e, 0xbe, 0x0b, 0xfd, 0x71, 0x8e, 0x9a, 0xeb, + 0x60, 0x17, 0x36, 0xa2, 0x44, 0x8c, 0xf3, 0x0e, 0x35, 0xf9, 0xf9, 0x91, 0x4d, 0xfe, 0x5d, 0x20, + 0xa6, 0x6d, 0xb2, 0x60, 0xca, 0x25, 0xde, 0x44, 0x3c, 0xf3, 0xb7, 0x31, 0xc0, 0x81, 0x29, 0x52, + 0x43, 0x96, 0xf2, 0xe4, 0xf9, 0x2e, 0xe5, 0xc2, 0xe8, 0x4b, 0x99, 0x3c, 0x82, 0x6b, 0x42, 0x95, + 0x1a, 0x9f, 0x38, 0xb0, 0x34, 0xfe, 0xff, 0x4f, 0x01, 0x5f, 0xc3, 0x61, 0x8c, 0x38, 0x1c, 0x83, + 0xcf, 0x8f, 0xe1, 0xd2, 0x26, 0x57, 0xae, 0x5b, 0xc3, 0x0f, 0x86, 0x95, 0x14, 0x1e, 0x4c, 0x95, + 0xe4, 0x4b, 0x8c, 0xf1, 0x65, 0xa8, 0xef, 0x59, 0xb4, 0x29, 0x0e, 0x82, 0x42, 0xb8, 0xc4, 0x76, + 0x36, 0x1b, 0x8a, 0x82, 0x11, 0xae, 0x34, 0x5b, 0x3d, 0x75, 0x46, 0x5b, 0xbd, 0x2e, 0xca, 0x3c, + 0xad, 0xd8, 0x91, 0xa0, 0x0c, 0x7e, 0x90, 0xcb, 0x5e, 0x49, 0x32, 0xe0, 0xa0, 0x8c, 0x38, 0x2a, + 0x0d, 0xd7, 0xec, 0x31, 0x2f, 0x8e, 0x75, 0x29, 0x71, 0x54, 0xa6, 0xf0, 0x60, 0xaa, 0x24, 0x77, + 0x52, 0xf6, 0xa9, 0x6e, 0xb1, 0xfd, 0x38, 0xe0, 0x4c, 0xdc, 0x49, 0xb9, 0x33, 0xc8, 0x82, 0x69, + 0x72, 0xe3, 0x98, 0xb7, 0x3f, 0xc8, 0xc0, 0xe5, 0x75, 0xaa, 0x4a, 0x2c, 0xdb, 0x4e, 0xd3, 0xb7, + 0x6b, 0xff, 0x47, 0xa3, 0xac, 0x3f, 0xd6, 0x00, 0xee, 0xec, 0xec, 0x6c, 0xab, 0x10, 0xb9, 0x09, + 0x39, 0xbd, 0xaf, 0x32, 0x1c, 0xa5, 0xa5, 0xb5, 0xd1, 0x2b, 0x59, 0xd1, 0xa4, 0xb7, 0x4a, 0x27, + 0xf4, 0xd9, 0x3e, 0x0a, 0x74, 0xf2, 0xff, 0x61, 0x52, 0x9d, 0x0d, 0x62, 0xac, 0x0a, 0x61, 0x45, + 0x41, 0x9d, 0x1f, 0xe8, 0xd3, 0xab, 0x3f, 0xcd, 0xc0, 0xd5, 0x0d, 0x9b, 0x51, 0xb7, 0xc1, 0x68, + 0x2f, 0x96, 0x94, 0x26, 0xbf, 0x11, 0xa9, 0xf5, 0xc9, 0xfe, 0xfe, 0xc2, 0xe9, 0x62, 0x76, 0x59, + 0x2f, 0xda, 0xa2, 0x4c, 0x0f, 0x77, 0x65, 0xd8, 0x16, 0x29, 0xf0, 0xf5, 0x21, 0xe7, 0xf5, 0xa8, + 0xa1, 0x32, 0x02, 0x8d, 0x91, 0x47, 0x23, 0xfd, 0x05, 0xf8, 0xca, 0x0b, 0x73, 0x31, 0x62, 0x1d, + 0x0a, 0x75, 0xe4, 0x5b, 0x30, 0xe1, 0x31, 0x9d, 0xf5, 0xfd, 0x0c, 0xdb, 0xee, 0x79, 0x2b, 0x16, + 0xe0, 0xe1, 0x01, 0x29, 0x9f, 0x51, 0x29, 0xad, 0xfe, 0x54, 0x83, 0xf9, 0x74, 0xc1, 0x4d, 0xd3, + 0x63, 0xe4, 0x1b, 0x03, 0xc3, 0x7e, 0xca, 0x54, 0x09, 0x97, 0x16, 0x83, 0x3e, 0xab, 0x14, 0x17, + 0xfc, 0x96, 0xc8, 0x90, 0x33, 0xc8, 0x9b, 0x8c, 0x76, 0x7d, 0x2f, 0xe1, 0x9d, 0x73, 0x7e, 0xf5, + 0xc8, 0xae, 0xe4, 0x5a, 0x50, 0x2a, 0xab, 0x7e, 0x94, 0x19, 0xf6, 0xca, 0x7c, 0x5a, 0x48, 0x27, + 0x5e, 0xf8, 0xb8, 0x3b, 0x5e, 0xe1, 0xa3, 0xde, 0x8f, 0xf4, 0x67, 0xb0, 0xfc, 0xf1, 0x9b, 0x83, + 0xe5, 0x8f, 0x77, 0xc6, 0x2f, 0x7f, 0x24, 0x46, 0x61, 0x68, 0x15, 0xe4, 0xc7, 0x19, 0xb8, 0x7e, + 0xd2, 0xaa, 0x21, 0xed, 0x60, 0x71, 0x6a, 0xe3, 0x5e, 0x87, 0x38, 0x71, 0x19, 0x92, 0x25, 0xc8, + 0xf7, 0xf6, 0x75, 0xcf, 0x37, 0xa7, 0xfe, 0xa9, 0x93, 0xdf, 0xe6, 0x8d, 0x4f, 0x8f, 0x2a, 0x25, + 0x69, 0x86, 0xc5, 0x23, 0x4a, 0x56, 0x6e, 0x58, 0xba, 0xd4, 0xf3, 0x42, 0xc7, 0x2e, 0x30, 0x2c, + 0x5b, 0xb2, 0x19, 0x7d, 0x3a, 0x61, 0x30, 0x21, 0x83, 0x25, 0x95, 0x4e, 0xde, 0x1c, 0xf9, 0x3d, + 0x52, 0x4a, 0x65, 0xe1, 0x4b, 0xa9, 0xb8, 0x5b, 0xe9, 0xaa, 0xfe, 0xc5, 0x25, 0xb8, 0x9a, 0x3e, + 0x27, 0xbc, 0xef, 0x07, 0xd4, 0xf5, 0x4c, 0xc7, 0x56, 0xa7, 0x4f, 0x58, 0x66, 0x95, 0xcd, 0xe8, + 0xd3, 0xc9, 0x4d, 0x28, 0xb8, 0xb4, 0x67, 0x99, 0x86, 0xee, 0xa9, 0xa0, 0x43, 0x64, 0x1f, 0x51, + 0xb5, 0x61, 0x40, 0x1d, 0x72, 0xf5, 0x23, 0xfb, 0xbf, 0x78, 0xf5, 0xe3, 0x4f, 0x35, 0xee, 0xcf, + 0xc9, 0x8c, 0xc3, 0x80, 0x80, 0x9a, 0x8b, 0xf3, 0xec, 0xd9, 0xcb, 0xd2, 0x2f, 0x1c, 0xa2, 0x10, + 0x87, 0xf7, 0x85, 0xfc, 0x40, 0x83, 0x72, 0x37, 0xe1, 0x30, 0x5e, 0xe0, 0xed, 0x99, 0xeb, 0xc7, + 0x47, 0x95, 0xf2, 0xd6, 0x10, 0x7d, 0x38, 0xb4, 0x27, 0xe4, 0xb7, 0xa0, 0xd4, 0xe3, 0xeb, 0xc2, + 0x63, 0xd4, 0x36, 0xa8, 0xaa, 0xd5, 0x8d, 0xbe, 0x9a, 0xb7, 0x43, 0xac, 0x06, 0x73, 0x75, 0x46, + 0xdb, 0x87, 0xf5, 0x19, 0x1e, 0xda, 0x45, 0x08, 0x18, 0xd5, 0x18, 0xbb, 0x73, 0xb3, 0x75, 0xd1, + 0x77, 0x6e, 0xbe, 0x97, 0x7e, 0xe7, 0x46, 0x3f, 0x67, 0x0b, 0xf9, 0xfc, 0xee, 0xcd, 0xf3, 0xbb, + 0x37, 0x9f, 0xd7, 0xdd, 0x9b, 0x9b, 0x50, 0xf0, 0x28, 0x63, 0xa6, 0xdd, 0xf6, 0xca, 0xb3, 0xb2, + 0x40, 0xc7, 0xb5, 0x36, 0x54, 0x1b, 0x06, 0x54, 0xf2, 0xf3, 0x50, 0x14, 0x29, 0xb6, 0x65, 0xb7, + 0xed, 0x95, 0xe7, 0x44, 0xa5, 0x4e, 0x9c, 0xe4, 0x0d, 0xbf, 0x11, 0x43, 0x3a, 0x79, 0x03, 0xa6, + 0x64, 0x29, 0x51, 0x1e, 0x41, 0xe2, 0x9e, 0x4c, 0xb1, 0x3e, 0xcb, 0x57, 0x70, 0x3d, 0xd2, 0x8e, + 0x31, 0x2e, 0x1e, 0xba, 0xd2, 0x20, 0x0f, 0x59, 0xbe, 0x1c, 0x0f, 0x5d, 0xc3, 0x0c, 0x25, 0x46, + 0xb8, 0xc8, 0xcb, 0x90, 0x65, 0x96, 0x57, 0xbe, 0x22, 0x98, 0x83, 0x10, 0x63, 0x67, 0xb3, 0x81, + 0xbc, 0x7d, 0xfc, 0x4b, 0x34, 0xff, 0xad, 0xc1, 0x4c, 0xe2, 0x1e, 0x07, 0xd7, 0xd9, 0x77, 0x2d, + 0x75, 0x52, 0x06, 0x3a, 0x77, 0x71, 0x13, 0x79, 0x3b, 0x79, 0xa4, 0xe2, 0x98, 0xcc, 0x98, 0xf6, + 0xe8, 0xfe, 0xf2, 0x4e, 0x83, 0x07, 0x2e, 0x03, 0x21, 0xcc, 0xad, 0xc4, 0xe8, 0x66, 0xe3, 0x79, + 0xd1, 0x93, 0x47, 0x38, 0x92, 0x1c, 0xc8, 0x9d, 0x26, 0x39, 0x50, 0xfd, 0x54, 0x83, 0xe2, 0x3d, + 0xbd, 0xd5, 0xd1, 0x1b, 0xa6, 0xdd, 0x21, 0xaf, 0xc2, 0xe4, 0x9e, 0xeb, 0x74, 0xa8, 0xeb, 0xa9, + 0xea, 0xaf, 0x28, 0x29, 0xd6, 0x65, 0x13, 0xfa, 0x34, 0x1e, 0x8f, 0x32, 0xa7, 0x67, 0x1a, 0xc9, + 0x78, 0x74, 0x87, 0x37, 0xa2, 0xa4, 0x91, 0x87, 0x72, 0xee, 0xb2, 0x63, 0xde, 0xc4, 0xdc, 0xd9, + 0x6c, 0xc8, 0x8a, 0xaa, 0x3f, 0xeb, 0xe4, 0xb5, 0x98, 0x7f, 0x55, 0x1c, 0xea, 0x11, 0x7d, 0x2f, + 0x03, 0x25, 0xf9, 0x6a, 0x32, 0x02, 0x3d, 0xcf, 0x97, 0x7b, 0x4b, 0x24, 0xcf, 0xbd, 0x7e, 0x97, + 0xba, 0xeb, 0xae, 0xd3, 0xef, 0xa9, 0x59, 0x8a, 0x26, 0x43, 0x42, 0x62, 0x90, 0x40, 0x0f, 0x9b, + 0xfc, 0xd1, 0xc9, 0x5d, 0xe0, 0xe8, 0xe4, 0x4f, 0x1c, 0x9d, 0xbf, 0xd4, 0xa0, 0xb8, 0x69, 0xb6, + 0xa8, 0x71, 0x68, 0x58, 0x94, 0x7c, 0x03, 0xca, 0x4d, 0x6a, 0x51, 0x46, 0xd7, 0x5d, 0xdd, 0xa0, + 0xdb, 0xd4, 0x35, 0x85, 0xa9, 0x77, 0xec, 0xa6, 0xf4, 0xc6, 0xf3, 0x41, 0xc6, 0xa2, 0xbc, 0x3a, + 0x84, 0x0f, 0x87, 0x22, 0x90, 0x0d, 0x98, 0x6a, 0x52, 0xcf, 0x74, 0x69, 0x73, 0x3b, 0xe2, 0x77, + 0xbf, 0xea, 0x2f, 0xe9, 0xd5, 0x08, 0xed, 0xe9, 0x51, 0x65, 0x7a, 0xdb, 0xec, 0x51, 0xcb, 0xb4, + 0xa9, 0x74, 0xc0, 0x63, 0xa2, 0xd5, 0x3c, 0x64, 0x37, 0x9d, 0x76, 0xf5, 0x77, 0xb3, 0x10, 0x9c, + 0xe1, 0xe4, 0xf7, 0x34, 0x28, 0xe9, 0xb6, 0xed, 0x30, 0x75, 0x38, 0xca, 0xf4, 0x3d, 0x8e, 0xed, + 0x2a, 0xd4, 0x96, 0x43, 0x50, 0x79, 0x52, 0x07, 0x07, 0x6a, 0x84, 0x82, 0x51, 0xdd, 0xa4, 0x9f, + 0x48, 0x46, 0x6f, 0x8d, 0xdf, 0x8b, 0x53, 0xa4, 0x9e, 0xe7, 0xbf, 0x06, 0xb3, 0xc9, 0xce, 0x9e, + 0xc5, 0x10, 0x8e, 0x93, 0xf6, 0xfa, 0x13, 0x0d, 0x0a, 0xbe, 0x31, 0x23, 0x2b, 0x90, 0xeb, 0x7b, + 0xd4, 0x3d, 0xdb, 0x1d, 0x44, 0x61, 0x01, 0x77, 0x3d, 0xea, 0xa2, 0x10, 0x26, 0xef, 0x40, 0xa1, + 0xa7, 0x7b, 0xde, 0x63, 0xc7, 0x6d, 0x2a, 0x33, 0x7b, 0x4a, 0x20, 0x79, 0x36, 0x2b, 0x51, 0x0c, + 0x40, 0xaa, 0x3f, 0x9a, 0x86, 0xd2, 0x7d, 0x9d, 0x99, 0x07, 0x54, 0xc4, 0xc3, 0x17, 0x13, 0x10, + 0x7d, 0x5f, 0x83, 0xab, 0xf1, 0xcc, 0xf5, 0x05, 0x46, 0x45, 0xf3, 0xc7, 0x47, 0x95, 0xab, 0x98, + 0xaa, 0x0d, 0x87, 0xf4, 0x42, 0xc4, 0x47, 0x03, 0x89, 0xf0, 0x8b, 0x8e, 0x8f, 0x1a, 0xc3, 0x14, + 0xe2, 0xf0, 0xbe, 0x3c, 0x8f, 0x8f, 0x46, 0x88, 0x8f, 0x2e, 0xfc, 0x9b, 0x84, 0xef, 0xa4, 0xc7, + 0x47, 0x0f, 0x46, 0xf7, 0x80, 0xc2, 0x1d, 0xf9, 0x3c, 0x28, 0x7a, 0x1e, 0x14, 0x7d, 0x5e, 0x41, + 0x51, 0x2f, 0x11, 0x14, 0x8d, 0x53, 0x8c, 0x50, 0x55, 0x7e, 0x89, 0x36, 0x2c, 0xb8, 0x1a, 0x3f, + 0x4c, 0xf9, 0xa3, 0x0c, 0x5c, 0x4e, 0xb1, 0x0e, 0xe4, 0x6d, 0x98, 0xf5, 0x98, 0xe3, 0xea, 0x6d, + 0x1a, 0x4e, 0xa8, 0x3c, 0xd0, 0xc4, 0x0d, 0xd0, 0x46, 0x82, 0x86, 0x03, 0xdc, 0xe4, 0x11, 0x80, + 0x6e, 0x18, 0xd4, 0xf3, 0xb6, 0x9c, 0xa6, 0xef, 0x97, 0xbd, 0xc5, 0xc3, 0x85, 0xe5, 0xa0, 0xf5, + 0xe9, 0x51, 0xe5, 0x4b, 0x69, 0x05, 0x23, 0xbf, 0x3f, 0x4c, 0xde, 0xf5, 0x0e, 0x05, 0x30, 0x02, + 0x49, 0x7e, 0x1d, 0x40, 0xde, 0xfe, 0x0e, 0xee, 0x29, 0x3e, 0x23, 0xab, 0x5f, 0xf3, 0x6f, 0x57, + 0xd7, 0x7e, 0xad, 0xaf, 0xdb, 0x8c, 0xaf, 0x0a, 0x71, 0x77, 0xf7, 0x41, 0x80, 0x82, 0x11, 0xc4, + 0xea, 0xdf, 0x67, 0xa0, 0xe0, 0xfb, 0x8b, 0x9f, 0x43, 0xdd, 0xa6, 0x1d, 0xab, 0xdb, 0x8c, 0xfe, + 0x11, 0x8a, 0xdf, 0xe5, 0xa1, 0x95, 0x1a, 0x27, 0x51, 0xa9, 0x59, 0x1f, 0x5f, 0xd5, 0xc9, 0xb5, + 0x99, 0x1f, 0x66, 0xe0, 0x92, 0xcf, 0xaa, 0xee, 0x27, 0x7f, 0x05, 0xa6, 0x5d, 0xaa, 0x37, 0xeb, + 0x3a, 0x33, 0xf6, 0xc5, 0xf4, 0xc9, 0xdb, 0xc9, 0x73, 0xc7, 0x47, 0x95, 0x69, 0x8c, 0x12, 0x30, + 0xce, 0x97, 0x76, 0xb1, 0x39, 0x33, 0xe6, 0xc5, 0xe6, 0xec, 0x59, 0x2e, 0x36, 0x13, 0x1d, 0x4a, + 0xbc, 0x47, 0x3b, 0x66, 0x97, 0x3a, 0x7d, 0xa6, 0x7c, 0x9b, 0xb3, 0xde, 0xbb, 0x15, 0x67, 0x2f, + 0x86, 0x30, 0x18, 0xc5, 0xac, 0xfe, 0x93, 0x06, 0x53, 0xe1, 0x78, 0x5d, 0x78, 0xf5, 0xaa, 0x15, + 0xaf, 0x5e, 0x2d, 0x8f, 0xbd, 0x1c, 0x86, 0xd4, 0xab, 0x7e, 0x90, 0x0f, 0x5f, 0x4b, 0x54, 0xa8, + 0xf6, 0x60, 0xde, 0x4c, 0xad, 0xda, 0x44, 0xac, 0x4d, 0x70, 0x15, 0x6d, 0x63, 0x28, 0x27, 0x9e, + 0x80, 0x42, 0xfa, 0x50, 0x38, 0xa0, 0x2e, 0x33, 0x0d, 0xea, 0xbf, 0xdf, 0xfa, 0x39, 0x7d, 0xe9, + 0x18, 0x8e, 0xe9, 0x03, 0xa5, 0x00, 0x03, 0x55, 0x64, 0x0f, 0xf2, 0xb4, 0xd9, 0xa6, 0xfe, 0xd5, + 0xf3, 0x37, 0xc7, 0xba, 0xf6, 0x1f, 0x8e, 0x27, 0x7f, 0xf2, 0x50, 0x42, 0x13, 0x0f, 0x8a, 0x96, + 0x1f, 0x65, 0xab, 0x75, 0x58, 0x1f, 0x59, 0x4f, 0x10, 0xaf, 0x87, 0x57, 0x41, 0x83, 0x26, 0x0c, + 0xf5, 0x90, 0x4e, 0xf0, 0x41, 0x43, 0xfe, 0x9c, 0x8c, 0xc7, 0x09, 0x9f, 0xcb, 0x79, 0x50, 0x7c, + 0xac, 0x33, 0xea, 0x76, 0x75, 0xb7, 0xa3, 0x7c, 0xe0, 0xd1, 0xdf, 0xf0, 0xa1, 0x8f, 0x14, 0xbe, + 0x61, 0xd0, 0x84, 0xa1, 0x9e, 0xea, 0x8f, 0x22, 0xd6, 0xea, 0xf3, 0x2e, 0x1f, 0xbe, 0x11, 0x2f, + 0x1f, 0x2e, 0x24, 0xcb, 0x87, 0x89, 0xfc, 0xc5, 0xd9, 0x0b, 0x88, 0x3a, 0x94, 0x2c, 0xdd, 0x63, + 0xbb, 0xbd, 0xa6, 0xce, 0x54, 0x22, 0xaf, 0xb4, 0xf4, 0x73, 0xa7, 0x33, 0x26, 0xdc, 0x3c, 0x85, + 0x2e, 0xee, 0x66, 0x08, 0x83, 0x51, 0xcc, 0xea, 0xbf, 0x6b, 0x30, 0x37, 0x50, 0x32, 0x26, 0xfb, + 0x30, 0x61, 0x0b, 0xa7, 0x7c, 0xec, 0xef, 0xf0, 0x22, 0xbe, 0xbd, 0x5c, 0x34, 0xaa, 0x41, 0xe1, + 0x13, 0x1b, 0x0a, 0xf4, 0x09, 0xa3, 0xae, 0xad, 0x5b, 0xea, 0x2c, 0x3d, 0x9f, 0x6f, 0xfe, 0x84, + 0x0b, 0x76, 0x5b, 0x21, 0x63, 0xa0, 0xa3, 0xfa, 0xb3, 0x0c, 0x94, 0x22, 0x7c, 0xcf, 0x4a, 0xf2, + 0x8a, 0xab, 0x88, 0x32, 0x3a, 0xdd, 0x75, 0x2d, 0x35, 0xd1, 0x91, 0xab, 0x88, 0x8a, 0x84, 0x9b, + 0x18, 0xe5, 0x23, 0x4b, 0x00, 0x5d, 0xdd, 0x63, 0xd4, 0x15, 0xb6, 0x31, 0x71, 0x01, 0x70, 0x2b, + 0xa0, 0x60, 0x84, 0x8b, 0xdc, 0x50, 0x19, 0x93, 0x5c, 0xfc, 0x03, 0x9a, 0x21, 0xe9, 0x90, 0xfc, + 0x39, 0xa4, 0x43, 0x48, 0x1b, 0x66, 0xfd, 0x5e, 0xfb, 0x54, 0xb5, 0x71, 0x4f, 0x09, 0x2c, 0xbd, + 0xcb, 0x04, 0x04, 0x0e, 0x80, 0x56, 0xff, 0x4a, 0x83, 0xe9, 0x98, 0x8b, 0x4c, 0x5e, 0x89, 0xde, + 0x77, 0x88, 0x24, 0x57, 0x63, 0xf7, 0x14, 0x5e, 0x83, 0x09, 0x39, 0x40, 0x6a, 0xe0, 0x83, 0x8d, + 0x28, 0x87, 0x10, 0x15, 0x95, 0x6f, 0x29, 0x95, 0x7d, 0x49, 0x6e, 0x29, 0x95, 0x9e, 0x41, 0x9f, + 0x4e, 0xbe, 0xc8, 0x9d, 0x7e, 0xd9, 0x3b, 0x35, 0xd2, 0xc1, 0xc1, 0xe0, 0xbf, 0x07, 0x06, 0x1c, + 0xd5, 0xef, 0xe7, 0x40, 0x7e, 0xad, 0xcb, 0xe5, 0x9a, 0xa6, 0x27, 0x13, 0xea, 0x9a, 0x48, 0xa8, + 0x07, 0x72, 0xab, 0xaa, 0x1d, 0x03, 0x0e, 0x72, 0x0d, 0xb2, 0x5d, 0xd3, 0x56, 0x79, 0x22, 0x91, + 0x96, 0xdd, 0x32, 0x6d, 0xe4, 0x6d, 0x82, 0xa4, 0x3f, 0x51, 0x37, 0x6f, 0x25, 0x49, 0x7f, 0x82, + 0xbc, 0x8d, 0x7b, 0x4b, 0x96, 0xe3, 0x74, 0xf6, 0x74, 0xa3, 0xe3, 0xa7, 0x5c, 0x73, 0xc2, 0xdb, + 0x11, 0xde, 0xd2, 0x66, 0x9c, 0x84, 0x49, 0x5e, 0x2e, 0x6e, 0x38, 0x8e, 0xd5, 0x74, 0x1e, 0xdb, + 0xbe, 0x78, 0x3e, 0x14, 0x5f, 0x89, 0x93, 0x30, 0xc9, 0x4b, 0x76, 0xe1, 0xa5, 0x0f, 0xa9, 0xeb, + 0xa8, 0x11, 0x6b, 0x58, 0x94, 0xf6, 0x7c, 0x98, 0x09, 0x01, 0xf3, 0x85, 0xe3, 0xa3, 0xca, 0x4b, + 0xef, 0xa5, 0xb3, 0xe0, 0x30, 0x59, 0x0e, 0xcb, 0x74, 0xb7, 0x4d, 0xd9, 0xb6, 0xeb, 0xf0, 0x60, + 0xc0, 0xb4, 0xdb, 0x3e, 0xec, 0x64, 0x08, 0xbb, 0x93, 0xce, 0x82, 0xc3, 0x64, 0xc9, 0x0a, 0xcc, + 0x49, 0x52, 0xc4, 0x09, 0x14, 0x77, 0x40, 0xa7, 0xe5, 0x45, 0x81, 0x9d, 0x24, 0x11, 0x07, 0xf9, + 0xb9, 0x7f, 0xe9, 0x67, 0xed, 0xb6, 0xa9, 0x2b, 0x26, 0x5a, 0x5c, 0xf7, 0x54, 0xfe, 0x25, 0x26, + 0x68, 0x38, 0xc0, 0x2d, 0x3e, 0x7a, 0x14, 0x05, 0x93, 0x87, 0x90, 0xb5, 0x9c, 0xb6, 0x32, 0x97, + 0xa3, 0xa7, 0xf1, 0x37, 0x9d, 0xb6, 0x5c, 0x14, 0x9b, 0x4e, 0x1b, 0x39, 0x22, 0x31, 0x20, 0xdf, + 0xd1, 0x5b, 0x1d, 0x5d, 0x59, 0xc7, 0xd1, 0x4f, 0xd4, 0xa0, 0xb8, 0xa3, 0xbe, 0xe4, 0xe3, 0x8f, + 0x28, 0xb1, 0x89, 0x01, 0x13, 0xfd, 0xa6, 0xf8, 0x77, 0x90, 0x71, 0xff, 0x80, 0x64, 0x77, 0x55, + 0xa8, 0x10, 0xa6, 0x5e, 0xfe, 0x46, 0x05, 0x5d, 0xfd, 0x87, 0x0c, 0xa8, 0xff, 0x24, 0x21, 0x7d, + 0x28, 0xb6, 0xfd, 0x2f, 0xe7, 0xd4, 0x98, 0xdd, 0x19, 0xe3, 0xb2, 0x76, 0xec, 0x1b, 0x3c, 0x59, + 0xaf, 0x0c, 0x1a, 0x31, 0xd4, 0x44, 0x68, 0x7c, 0x2c, 0x57, 0xc7, 0x1c, 0x4b, 0xa9, 0x6e, 0x70, + 0x34, 0x75, 0xc8, 0xed, 0x33, 0xd6, 0x53, 0x63, 0x39, 0xfa, 0x47, 0xa4, 0xe1, 0xa5, 0x49, 0x99, + 0x19, 0xe7, 0xcf, 0x28, 0xa0, 0xab, 0x5d, 0x50, 0xce, 0x08, 0x31, 0x62, 0xdf, 0xcb, 0xca, 0x8a, + 0xc7, 0xe2, 0xe9, 0x5c, 0x84, 0xe0, 0x5b, 0xcf, 0xc8, 0x67, 0x34, 0xa9, 0x1f, 0xc6, 0x56, 0xff, + 0x25, 0x03, 0xd9, 0x9d, 0xcd, 0x86, 0xbc, 0x15, 0x2e, 0xd2, 0x57, 0xb4, 0xd1, 0x31, 0x7b, 0x0f, + 0xa8, 0x6b, 0xb6, 0x0e, 0x95, 0x3d, 0x8c, 0xdc, 0x0a, 0x4f, 0x72, 0x60, 0x8a, 0x14, 0x79, 0x1f, + 0xa6, 0x0c, 0x7d, 0x85, 0xba, 0x4c, 0x1e, 0x2b, 0x67, 0x4b, 0xf0, 0x8b, 0x1a, 0xf3, 0xca, 0x72, + 0x28, 0x8e, 0x31, 0x30, 0xb2, 0x0b, 0x60, 0x84, 0xd0, 0xd9, 0xb3, 0x40, 0xcb, 0x0f, 0x84, 0x43, + 0xe0, 0x08, 0x10, 0x41, 0x28, 0x76, 0x38, 0xab, 0x40, 0xcd, 0x9d, 0x05, 0x55, 0x2c, 0xca, 0x7b, + 0xbe, 0x2c, 0x86, 0x30, 0xd5, 0x9f, 0x68, 0x90, 0xdd, 0x5d, 0x5d, 0x23, 0x0e, 0x14, 0x83, 0xab, + 0x4c, 0x6a, 0x4f, 0xd4, 0xc7, 0xcf, 0x6d, 0x4b, 0xc5, 0xc1, 0x23, 0x86, 0x3a, 0xc8, 0x3e, 0x4c, + 0xee, 0xf5, 0x4d, 0x8b, 0x99, 0xb6, 0x48, 0xfe, 0x8d, 0x13, 0x4b, 0xfa, 0x9f, 0xdc, 0xaa, 0x4a, + 0xaa, 0x44, 0x45, 0x1f, 0xbe, 0xfa, 0x2d, 0x50, 0xb6, 0x80, 0xc7, 0x08, 0x17, 0xf1, 0x92, 0x41, + 0x8c, 0x90, 0xf6, 0xa2, 0xd5, 0xbf, 0xcb, 0xc0, 0x84, 0xfa, 0x2f, 0xa6, 0x8b, 0x4f, 0x0c, 0xd1, + 0x58, 0x62, 0x68, 0x65, 0xcc, 0x7f, 0x27, 0x19, 0x9a, 0x16, 0xea, 0x26, 0xd2, 0x42, 0xe3, 0xfe, + 0x0d, 0xca, 0x33, 0x92, 0x42, 0xff, 0xa8, 0xc1, 0x25, 0xc9, 0xb8, 0x61, 0x7b, 0x4c, 0xb7, 0x0d, + 0xf1, 0xa7, 0x55, 0x07, 0xa2, 0x65, 0xec, 0x30, 0x4b, 0x45, 0xe8, 0xe2, 0xcc, 0x90, 0xbf, 0x51, + 0x41, 0x73, 0xb7, 0x6b, 0xdf, 0xf1, 0x98, 0xf8, 0xb2, 0x3c, 0x13, 0x77, 0xd7, 0xee, 0xa8, 0x76, + 0x0c, 0x38, 0x92, 0x7e, 0x60, 0x7e, 0xb8, 0x1f, 0x58, 0xfd, 0x73, 0x0d, 0xa6, 0xa2, 0x7f, 0x00, + 0x33, 0x7a, 0x8e, 0x2b, 0x91, 0x62, 0xca, 0x5c, 0x40, 0x8a, 0xe9, 0x13, 0x0d, 0xc0, 0xef, 0xec, + 0x85, 0x27, 0x98, 0x9a, 0xf1, 0x04, 0xd3, 0xd8, 0xd3, 0x9a, 0x9e, 0x5e, 0xfa, 0xeb, 0x9c, 0xff, + 0x4a, 0x22, 0xb9, 0xf4, 0x91, 0x06, 0x97, 0xf4, 0x58, 0xc2, 0x46, 0xbd, 0xd9, 0xb9, 0xe5, 0x7f, + 0xae, 0xaa, 0x6e, 0x24, 0xfe, 0x9d, 0x0d, 0x13, 0x6a, 0xc9, 0x2d, 0x98, 0xea, 0xa9, 0xb0, 0xfd, + 0x7e, 0xb8, 0xea, 0x82, 0x22, 0xd4, 0x76, 0x84, 0x86, 0x31, 0xce, 0x67, 0x24, 0xc8, 0xb2, 0xe7, + 0x92, 0x20, 0x8b, 0x56, 0xa1, 0x73, 0x27, 0x56, 0xa1, 0x6d, 0x28, 0xb6, 0x5c, 0xa7, 0x2b, 0x72, + 0x50, 0xea, 0x5f, 0x5b, 0xc6, 0xcc, 0x6b, 0x05, 0x46, 0x76, 0xcd, 0xc7, 0xc5, 0x50, 0x05, 0x3f, + 0x4d, 0x98, 0x23, 0xb5, 0x4d, 0x9c, 0x87, 0xb6, 0x60, 0xeb, 0xee, 0x48, 0x54, 0xf4, 0xe1, 0xab, + 0xff, 0x9c, 0xf1, 0xb7, 0x6e, 0x23, 0x71, 0x8d, 0x5b, 0x1b, 0x72, 0x8d, 0x5b, 0x7d, 0x89, 0x13, + 0xcd, 0xc2, 0xbc, 0x06, 0x13, 0x2e, 0xd5, 0x3d, 0xc7, 0x56, 0xdf, 0xb4, 0x05, 0x86, 0x0f, 0x45, + 0x2b, 0x2a, 0x6a, 0x34, 0x5b, 0x93, 0x79, 0x46, 0xb6, 0xe6, 0x8b, 0x91, 0xb9, 0x91, 0x59, 0xea, + 0x60, 0x9b, 0xa5, 0xcc, 0x8f, 0x08, 0x44, 0x55, 0x01, 0x35, 0x9f, 0x0c, 0x44, 0x55, 0x91, 0x33, + 0xe0, 0x20, 0x4d, 0x98, 0xb2, 0x74, 0x8f, 0x89, 0xa0, 0xa3, 0xb9, 0xcc, 0x46, 0x48, 0x05, 0x05, + 0x2b, 0x78, 0x33, 0x82, 0x83, 0x31, 0xd4, 0xea, 0x77, 0x35, 0x08, 0xb3, 0x6c, 0x67, 0x0c, 0x79, + 0xbf, 0x0e, 0x85, 0xae, 0xfe, 0x64, 0x95, 0x5a, 0xfa, 0xe1, 0x38, 0x7f, 0x6f, 0xb1, 0xa5, 0x30, + 0x30, 0x40, 0xab, 0xd7, 0x3e, 0xfe, 0x6c, 0xe1, 0x85, 0x4f, 0x3e, 0x5b, 0x78, 0xe1, 0xd3, 0xcf, + 0x16, 0x5e, 0xf8, 0xed, 0xe3, 0x05, 0xed, 0xe3, 0xe3, 0x05, 0xed, 0x93, 0xe3, 0x05, 0xed, 0xd3, + 0xe3, 0x05, 0xed, 0x27, 0xc7, 0x0b, 0xda, 0x77, 0xfe, 0x75, 0xe1, 0x85, 0xf7, 0x0a, 0xfe, 0xf2, + 0xf9, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xed, 0xc9, 0x9b, 0x44, 0x2d, 0x53, 0x00, 0x00, } func (m *AbstractVertex) Marshal() (dAtA []byte, err error) { @@ -5292,8 +5292,20 @@ func (m *Watermark) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.MaxDelay != nil { + { + size, err := m.MaxDelay.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintGenerated(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } i-- - if m.Propagate { + if m.Disabled { dAtA[i] = 1 } else { dAtA[i] = 0 @@ -6563,6 +6575,10 @@ func (m *Watermark) Size() (n int) { var l int _ = l n += 2 + if m.MaxDelay != nil { + l = m.MaxDelay.Size() + n += 1 + l + sovGenerated(uint64(l)) + } return n } @@ -7491,7 +7507,8 @@ func (this *Watermark) String() string { return "nil" } s := strings.Join([]string{`&Watermark{`, - `Propagate:` + fmt.Sprintf("%v", this.Propagate) + `,`, + `Disabled:` + fmt.Sprintf("%v", this.Disabled) + `,`, + `MaxDelay:` + strings.Replace(fmt.Sprintf("%v", this.MaxDelay), "Duration", "v11.Duration", 1) + `,`, `}`, }, "") return s @@ -18463,7 +18480,7 @@ func (m *Watermark) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Propagate", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Disabled", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -18480,7 +18497,43 @@ func (m *Watermark) Unmarshal(dAtA []byte) error { break } } - m.Propagate = bool(v != 0) + m.Disabled = bool(v != 0) + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxDelay", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthGenerated + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthGenerated + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.MaxDelay == nil { + m.MaxDelay = &v11.Duration{} + } + if err := m.MaxDelay.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/pkg/apis/numaflow/v1alpha1/generated.proto b/pkg/apis/numaflow/v1alpha1/generated.proto index de26295be6..9678fcf102 100644 --- a/pkg/apis/numaflow/v1alpha1/generated.proto +++ b/pkg/apis/numaflow/v1alpha1/generated.proto @@ -731,7 +731,7 @@ message PipelineSpec { // Watermark enables watermark progression across the entire pipeline. Updating this after the pipeline has been // created will have no impact and will be ignored. To make the pipeline honor any changes to the setting, the pipeline // should be recreated. - // +kubebuilder:default={"propagate": false} + // +kubebuilder:default={"disabled": false} // +optional optional Watermark watermark = 6; } @@ -983,9 +983,14 @@ message VertexStatus { } message Watermark { - // Propagate toggles the watermark propagation. + // Disabled toggles the watermark propagation, defaults to false. // +kubebuilder:default=false // +optional - optional bool propagate = 1; + optional bool disabled = 1; + + // Maximum delay allowed for watermark calculation, defaults to "0s", which means no delay. + // +kubebuilder:default="0s" + // +optional + optional k8s.io.apimachinery.pkg.apis.meta.v1.Duration maxDelay = 2; } diff --git a/pkg/apis/numaflow/v1alpha1/pipeline_types.go b/pkg/apis/numaflow/v1alpha1/pipeline_types.go index 5b8c52f219..7694aa7297 100644 --- a/pkg/apis/numaflow/v1alpha1/pipeline_types.go +++ b/pkg/apis/numaflow/v1alpha1/pipeline_types.go @@ -292,16 +292,20 @@ type PipelineSpec struct { // Watermark enables watermark progression across the entire pipeline. Updating this after the pipeline has been // created will have no impact and will be ignored. To make the pipeline honor any changes to the setting, the pipeline // should be recreated. - // +kubebuilder:default={"propagate": false} + // +kubebuilder:default={"disabled": false} // +optional Watermark Watermark `json:"watermark,omitempty" protobuf:"bytes,6,opt,name=watermark"` } type Watermark struct { - // Propagate toggles the watermark propagation. + // Disabled toggles the watermark propagation, defaults to false. // +kubebuilder:default=false // +optional - Propagate bool `json:"propagate,omitempty" protobuf:"bytes,1,opt,name=propagate"` + Disabled bool `json:"disabled,omitempty" protobuf:"bytes,1,opt,name=disabled"` + // Maximum delay allowed for watermark calculation, defaults to "0s", which means no delay. + // +kubebuilder:default="0s" + // +optional + MaxDelay *metav1.Duration `json:"maxDelay,omitempty" protobuf:"bytes,2,opt,name=maxDelay"` } type PipelineLimits struct { diff --git a/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go index f2f999a980..5dd386f34d 100644 --- a/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go @@ -1136,7 +1136,7 @@ func (in *PipelineSpec) DeepCopyInto(out *PipelineSpec) { *out = new(PipelineLimits) (*in).DeepCopyInto(*out) } - out.Watermark = in.Watermark + in.Watermark.DeepCopyInto(&out.Watermark) return } @@ -1615,6 +1615,11 @@ func (in *VertexStatus) DeepCopy() *VertexStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Watermark) DeepCopyInto(out *Watermark) { *out = *in + if in.MaxDelay != nil { + in, out := &in.MaxDelay, &out.MaxDelay + *out = new(metav1.Duration) + **out = **in + } return } diff --git a/pkg/daemon/server/service/pipeline_watermark_query.go b/pkg/daemon/server/service/pipeline_watermark_query.go index 538996f936..d90724911e 100644 --- a/pkg/daemon/server/service/pipeline_watermark_query.go +++ b/pkg/daemon/server/service/pipeline_watermark_query.go @@ -16,6 +16,8 @@ import ( "github.com/numaproj/numaflow/pkg/watermark/store/jetstream" ) +// TODO: This is not right, pending fix. + // watermarkFetchers used to store watermark metadata for propagation type watermarkFetchers struct { fetchMap map[string]fetch.Fetcher @@ -33,7 +35,7 @@ func newVertexWatermarkFetcher(pipeline *v1alpha1.Pipeline) (*watermarkFetchers, var wmFetcher = new(watermarkFetchers) var fromBufferName string - wmFetcher.isWatermarkEnabled = pipeline.Spec.Watermark.Propagate + wmFetcher.isWatermarkEnabled = !pipeline.Spec.Watermark.Disabled if !wmFetcher.isWatermarkEnabled { return wmFetcher, nil } @@ -84,7 +86,7 @@ func createWatermarkFetcher(ctx context.Context, pipelineName string, fromBuffer return nil, err } var fetchWmWatchers = generic.BuildFetchWMWatchers(hbWatch, otWatch) - fetchWatermark := generic.NewGenericFetch(ctx, vertexName, fetchWmWatchers) + fetchWatermark := generic.NewGenericEdgeFetch(ctx, vertexName, fetchWmWatchers) return fetchWatermark, nil } diff --git a/pkg/isb/forward/forward.go b/pkg/isb/forward/forward.go index eac1df5be6..c042fe96f2 100644 --- a/pkg/isb/forward/forward.go +++ b/pkg/isb/forward/forward.go @@ -10,7 +10,6 @@ import ( "time" "github.com/numaproj/numaflow/pkg/watermark/fetch" - "github.com/numaproj/numaflow/pkg/watermark/processor" "github.com/numaproj/numaflow/pkg/watermark/publish" "go.uber.org/zap" @@ -137,10 +136,8 @@ func (isdf *InterStepDataForward) Start() <-chan struct{} { } // stop watermark publisher if watermarking is enabled - if isdf.publishWatermark != nil { - for _, publisher := range isdf.publishWatermark { - publisher.StopPublisher() - } + for _, publisher := range isdf.publishWatermark { + publisher.StopPublisher() } close(stopped) }() @@ -180,9 +177,13 @@ func (isdf *InterStepDataForward) forwardAChunk(ctx context.Context) { // fetch watermark if available // TODO: make it async (concurrent and wait later) // let's track only the last element's watermark - var processorWM processor.Watermark - if isdf.fetchWatermark != nil { - processorWM = isdf.fetchWatermark.GetWatermark(readMessages[len(readMessages)-1].ReadOffset) + processorWM := isdf.fetchWatermark.GetWatermark(readMessages[len(readMessages)-1].ReadOffset) + if isdf.opts.isFromSourceVertex { // Set late data at source level + for _, m := range readMessages { + if processorWM.After(m.EventTime) { + m.IsLate = true + } + } } // create space for writeMessages specific to each step as we could forward to all the steps too. @@ -250,11 +251,11 @@ func (isdf *InterStepDataForward) forwardAChunk(ctx context.Context) { // forward the highest watermark to all the edges to avoid idle edge problem // TODO: sort and get the highest value - if isdf.publishWatermark != nil { - // TODO: Should also publish to those edges without writing (fall out of conditional forwarding)? - for edgeName, offsets := range writeOffsets { - if len(offsets) > 0 { - isdf.publishWatermark[edgeName].PublishWatermark(processorWM, offsets[len(offsets)-1]) + // TODO: Should also publish to those edges without writing (fall out of conditional forwarding)? + for edgeName, offsets := range writeOffsets { + if len(offsets) > 0 { + if publisher, ok := isdf.publishWatermark[edgeName]; ok { + publisher.PublishWatermark(processorWM, offsets[len(offsets)-1]) } } } diff --git a/pkg/isb/forward/options.go b/pkg/isb/forward/options.go index d163bd83c0..c019442440 100644 --- a/pkg/isb/forward/options.go +++ b/pkg/isb/forward/options.go @@ -14,6 +14,8 @@ type options struct { udfConcurrency int // retryInterval is the time.Duration to sleep before retrying retryInterval time.Duration + // isFromSourceVertex indicates if the fromStep is a source + isFromSourceVertex bool // logger is used to pass the logger variable logger *zap.SugaredLogger } @@ -51,3 +53,11 @@ func WithLogger(l *zap.SugaredLogger) Option { return nil } } + +// FromSourceVertex indicates it reads from a buffer written by a source vertex +func FromSourceVertex() Option { + return func(o *options) error { + o.isFromSourceVertex = true + return nil + } +} diff --git a/pkg/isb/message.go b/pkg/isb/message.go index 5970c0d268..517766af4e 100644 --- a/pkg/isb/message.go +++ b/pkg/isb/message.go @@ -10,9 +10,8 @@ type PaneInfo struct { EventTime time.Time StartTime time.Time EndTime time.Time - // IsWindow is used to represent whether we have applied window operator. - // ATM we do not support window as it requires an implicit reduce in the downstream. - IsWindow bool + // IsLate is used to indicate if it's a late data . + IsLate bool } // Header is the header of the message diff --git a/pkg/isb/stores/jetstream/reader.go b/pkg/isb/stores/jetstream/reader.go index 2c49a16fee..8127a7026d 100644 --- a/pkg/isb/stores/jetstream/reader.go +++ b/pkg/isb/stores/jetstream/reader.go @@ -255,8 +255,8 @@ func (jr *jetStreamReader) Ack(_ context.Context, offsets []isb.Offset) []error func convert2IsbMsgHeader(header nats.Header) isb.Header { r := isb.Header{} - if header.Get(_window) == "1" { - r.IsWindow = true + if header.Get(_late) == "1" { + r.IsLate = true } if x := header.Get(_id); x != "" { r.ID = x diff --git a/pkg/isb/stores/jetstream/writer.go b/pkg/isb/stores/jetstream/writer.go index 855fb1999e..17e54e0685 100644 --- a/pkg/isb/stores/jetstream/writer.go +++ b/pkg/isb/stores/jetstream/writer.go @@ -261,7 +261,7 @@ func (w *writeOffset) AckIt() error { const ( _key = "k" _id = "i" - _window = "w" + _late = "l" _eventTime = "pev" _startTime = "ps" _endTime = "pen" @@ -271,10 +271,10 @@ func convert2NatsMsgHeader(header isb.Header) nats.Header { r := nats.Header{} r.Add(_id, header.ID) r.Add(_key, string(header.Key)) - if header.IsWindow { - r.Add(_window, "1") + if header.IsLate { + r.Add(_late, "1") } else { - r.Add(_window, "0") + r.Add(_late, "0") } if !header.EventTime.IsZero() { r.Add(_eventTime, fmt.Sprint(header.EventTime.UnixMilli())) diff --git a/pkg/isbsvc/jetstream_service.go b/pkg/isbsvc/jetstream_service.go index 1775d3e991..e945feb99d 100644 --- a/pkg/isbsvc/jetstream_service.go +++ b/pkg/isbsvc/jetstream_service.go @@ -178,7 +178,6 @@ func (jss *jetStreamSvc) DeleteBuffers(ctx context.Context, buffers []dfv1.Buffe } func (jss *jetStreamSvc) ValidateBuffers(ctx context.Context, buffers []dfv1.Buffer) error { - log := logging.FromContext(ctx) nc, err := jsclient.NewInClusterJetStreamClient().Connect(ctx) if err != nil { return fmt.Errorf("failed to get an in-cluster nats connection, %w", err) @@ -198,14 +197,12 @@ func (jss *jetStreamSvc) ValidateBuffers(ctx context.Context, buffers []dfv1.Buf otBucket := JetStreamOTBucket(jss.pipelineName, buffer.Name) if _, err := js.KeyValue(otBucket); err != nil { - // TODO: throw an error - log.Warnw("Failed to query bucket", zap.String("bucket", otBucket), zap.Error(err)) + return fmt.Errorf("failed to query OT bucket %q, %w", otBucket, err) } procBucket := JetStreamProcessorBucket(jss.pipelineName, buffer.Name) if _, err := js.KeyValue(procBucket); err != nil { - // TODO: throw an error - log.Warnw("Failed to query bucket", zap.String("bucket", procBucket), zap.Error(err)) + return fmt.Errorf("failed to query processor bucket %q, %w", procBucket, err) } } return nil diff --git a/pkg/shared/util/watermark.go b/pkg/shared/util/watermark.go index 164e1623cf..620fa27c17 100644 --- a/pkg/shared/util/watermark.go +++ b/pkg/shared/util/watermark.go @@ -1,15 +1,19 @@ package util import ( - "os" + "time" dfv1 "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1" ) // IsWatermarkEnabled returns true of watermark has been enabled. func IsWatermarkEnabled() bool { - if val, ok := os.LookupEnv(dfv1.EnvWatermarkOn); ok && val == "true" { - return true - } - return false + return LookupEnvStringOr(dfv1.EnvWatermarkDisabled, "false") != "true" +} + +// GetWatermarkMaxDelay returns user configured watermark max delay +func GetWatermarkMaxDelay() time.Duration { + // The ENV variable is populated by the controller from validated user input, should not be invalid format. + d, _ := time.ParseDuration(LookupEnvStringOr(dfv1.EnvWatermarkMaxDelay, "0s")) + return d } diff --git a/pkg/sources/generator/tickgen.go b/pkg/sources/generator/tickgen.go index 89a368bd0a..4e0483b841 100644 --- a/pkg/sources/generator/tickgen.go +++ b/pkg/sources/generator/tickgen.go @@ -84,17 +84,10 @@ type memgen struct { vertexInstance *dfv1.VertexInstance // source watermark publisher sourcePublishWM publish.Publisher - // TODO: delete this watermark progressor - progressor *watermark logger *zap.SugaredLogger } -type watermark struct { - sourcePublish publish.Publisher - wmProgressor generic.Progressor -} - type Option func(*memgen) error // WithLogger is used to return logger information @@ -137,12 +130,8 @@ func NewMemGen(vertexInstance *dfv1.VertexInstance, pipelineName: vertexInstance.Vertex.Spec.PipelineName, genfn: recordGenerator, vertexInstance: vertexInstance, - progressor: &watermark{ - sourcePublish: nil, - wmProgressor: nil, - }, - srcchan: make(chan record, rpu*5), - readTimeout: 3 * time.Second, // default timeout + srcchan: make(chan record, rpu*5), + readTimeout: 3 * time.Second, // default timeout } for _, o := range opts { @@ -165,7 +154,7 @@ func NewMemGen(vertexInstance *dfv1.VertexInstance, destinations[w.GetName()] = w } - forwardOpts := []forward.Option{forward.WithLogger(gensrc.logger)} + forwardOpts := []forward.Option{forward.FromSourceVertex(), forward.WithLogger(gensrc.logger)} if x := vertexInstance.Vertex.Spec.Limits; x != nil { if x.ReadBatchSize != nil { forwardOpts = append(forwardOpts, forward.WithReadBatchSize(int64(*x.ReadBatchSize))) diff --git a/pkg/sources/generator/watermark.go b/pkg/sources/generator/watermark.go index 1655585251..a639b9e7fd 100644 --- a/pkg/sources/generator/watermark.go +++ b/pkg/sources/generator/watermark.go @@ -3,6 +3,7 @@ package generator import ( "fmt" + sharedutil "github.com/numaproj/numaflow/pkg/shared/util" "github.com/numaproj/numaflow/pkg/watermark/generic" "github.com/numaproj/numaflow/pkg/watermark/processor" "github.com/numaproj/numaflow/pkg/watermark/publish" @@ -12,5 +13,5 @@ func (mg *memgen) buildSourceWatermarkPublisher(publishWMStores *generic.Publish // for tickgen, it can be the name of the replica entityName := fmt.Sprintf("%s-%d", mg.vertexInstance.Vertex.Name, mg.vertexInstance.Replica) processorEntity := processor.NewProcessorEntity(entityName) - return publish.NewPublish(mg.lifecycleCtx, processorEntity, publishWMStores.HBStore, publishWMStores.OTStore) + return publish.NewPublish(mg.lifecycleCtx, processorEntity, publishWMStores.HBStore, publishWMStores.OTStore, publish.IsSource(), publish.WithDelay(sharedutil.GetWatermarkMaxDelay())) } diff --git a/pkg/sources/generator/watermark_test.go b/pkg/sources/generator/watermark_test.go index 6a5e412a95..c83957458b 100644 --- a/pkg/sources/generator/watermark_test.go +++ b/pkg/sources/generator/watermark_test.go @@ -22,8 +22,8 @@ func TestWatermark(t *testing.T) { // for use by the buffer reader on the other side of the stream ctx := context.Background() - _ = os.Setenv(dfv1.EnvWatermarkOn, "true") - defer func() { _ = os.Unsetenv(dfv1.EnvWatermarkOn) }() + _ = os.Setenv(dfv1.EnvWatermarkDisabled, "false") + defer func() { _ = os.Unsetenv(dfv1.EnvWatermarkDisabled) }() dest := simplebuffer.NewInMemoryBuffer("writer", 1000) vertex := &dfv1.Vertex{ObjectMeta: v1.ObjectMeta{ diff --git a/pkg/sources/http/http.go b/pkg/sources/http/http.go index 45deb0d0c2..8854aaf5e2 100644 --- a/pkg/sources/http/http.go +++ b/pkg/sources/http/http.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net/http" + "strconv" "time" "github.com/google/uuid" @@ -19,6 +20,10 @@ import ( sharedtls "github.com/numaproj/numaflow/pkg/shared/tls" sharedutil "github.com/numaproj/numaflow/pkg/shared/util" "github.com/numaproj/numaflow/pkg/udf/applier" + "github.com/numaproj/numaflow/pkg/watermark/fetch" + "github.com/numaproj/numaflow/pkg/watermark/generic" + "github.com/numaproj/numaflow/pkg/watermark/processor" + "github.com/numaproj/numaflow/pkg/watermark/publish" ) type httpSource struct { @@ -31,7 +36,11 @@ type httpSource struct { logger *zap.SugaredLogger forwarder *forward.InterStepDataForward - shutdown func(context.Context) error + // source watermark publisher + sourcePublishWM publish.Publisher + // context cancel function + cancelfn context.CancelFunc + shutdown func(context.Context) error } type Option func(*httpSource) error @@ -59,10 +68,10 @@ func WithBufferSize(s int) Option { } } -func New(vertex *dfv1.Vertex, writers []isb.BufferWriter, opts ...Option) (*httpSource, error) { +func New(vertexInstance *dfv1.VertexInstance, writers []isb.BufferWriter, fetchWM fetch.Fetcher, publishWM map[string]publish.Publisher, publishWMStores *generic.PublishWMStores, opts ...Option) (*httpSource, error) { h := &httpSource{ - name: vertex.Spec.Name, - pipelineName: vertex.Spec.PipelineName, + name: vertexInstance.Vertex.Spec.Name, + pipelineName: vertexInstance.Vertex.Spec.PipelineName, ready: false, bufferSize: 1000, // default size readTimeout: 1 * time.Second, // default timeout @@ -79,7 +88,7 @@ func New(vertex *dfv1.Vertex, writers []isb.BufferWriter, opts ...Option) (*http h.messages = make(chan *isb.ReadMessage, h.bufferSize) auth := "" - if x := vertex.Spec.Source.HTTP.Auth; x != nil && x.Token != nil { + if x := vertexInstance.Vertex.Spec.Source.HTTP.Auth; x != nil && x.Token != nil { if s, err := sharedutil.GetSecretFromVolume(x.Token); err != nil { return nil, fmt.Errorf("failed to get auth token, %w", err) } else { @@ -95,7 +104,7 @@ func New(vertex *dfv1.Vertex, writers []isb.BufferWriter, opts ...Option) (*http } w.WriteHeader(204) }) - mux.HandleFunc("/vertices/"+vertex.Spec.Name, func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/vertices/"+vertexInstance.Vertex.Spec.Name, func(w http.ResponseWriter, r *http.Request) { if auth != "" && r.Header.Get("Authorization") != "Bearer "+auth { w.WriteHeader(403) _, _ = w.Write([]byte("403 forbidden\n")) @@ -109,7 +118,7 @@ func New(vertex *dfv1.Vertex, writers []isb.BufferWriter, opts ...Option) (*http msg, err := io.ReadAll(r.Body) _ = r.Body.Close() if err != nil { - w.WriteHeader(400) + w.WriteHeader(http.StatusBadRequest) _, _ = w.Write([]byte(err.Error())) return } @@ -118,10 +127,20 @@ func New(vertex *dfv1.Vertex, writers []isb.BufferWriter, opts ...Option) (*http if id == "" { id = uuid.New().String() } + eventTime := time.Now() + if x := r.Header.Get(dfv1.KeyMetaEventTime); x != "" { + i, err := strconv.ParseInt(x, 10, 64) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(err.Error())) + return + } + eventTime = time.Unix(i, 0) + } m := &isb.ReadMessage{ Message: isb.Message{ Header: isb.Header{ - PaneInfo: isb.PaneInfo{EventTime: time.Now()}, + PaneInfo: isb.PaneInfo{EventTime: eventTime}, ID: id, }, Body: isb.Body{ @@ -131,7 +150,7 @@ func New(vertex *dfv1.Vertex, writers []isb.BufferWriter, opts ...Option) (*http ReadOffset: isb.SimpleOffset(func() string { return id }), } h.messages <- m - w.WriteHeader(204) + w.WriteHeader(http.StatusNoContent) }) cer, err := sharedtls.GenerateX509KeyPair() if err != nil { @@ -156,18 +175,23 @@ func New(vertex *dfv1.Vertex, writers []isb.BufferWriter, opts ...Option) (*http destinations[w.GetName()] = w } - forwardOpts := []forward.Option{forward.WithLogger(h.logger)} - if x := vertex.Spec.Limits; x != nil { + forwardOpts := []forward.Option{forward.FromSourceVertex(), forward.WithLogger(h.logger)} + if x := vertexInstance.Vertex.Spec.Limits; x != nil { if x.ReadBatchSize != nil { forwardOpts = append(forwardOpts, forward.WithReadBatchSize(int64(*x.ReadBatchSize))) } } - forwarder, err := forward.NewInterStepDataForward(vertex, h, destinations, forward.All, applier.Terminal, nil, nil, forwardOpts...) + forwarder, err := forward.NewInterStepDataForward(vertexInstance.Vertex, h, destinations, forward.All, applier.Terminal, fetchWM, publishWM, forwardOpts...) if err != nil { h.logger.Errorw("Error instantiating the forwarder", zap.Error(err)) return nil, err } h.forwarder = forwarder + ctx, cancel := context.WithCancel(context.Background()) + h.cancelfn = cancel + entityName := fmt.Sprintf("%s-%d", vertexInstance.Vertex.Name, vertexInstance.Replica) + processorEntity := processor.NewProcessorEntity(entityName) + h.sourcePublishWM = publish.NewPublish(ctx, processorEntity, publishWMStores.HBStore, publishWMStores.OTStore, publish.IsSource(), publish.WithDelay(sharedutil.GetWatermarkMaxDelay())) return h, nil } @@ -177,18 +201,26 @@ func (h *httpSource) GetName() string { func (h *httpSource) Read(ctx context.Context, count int64) ([]*isb.ReadMessage, error) { msgs := []*isb.ReadMessage{} + var latest time.Time timeout := time.After(h.readTimeout) +loop: for i := int64(0); i < count; i++ { select { case m := <-h.messages: + if latest.IsZero() || m.EventTime.After(latest) { + latest = m.EventTime + } msgs = append(msgs, m) + httpSourceReadCount.With(map[string]string{metricspkg.LabelVertex: h.name, metricspkg.LabelPipeline: h.pipelineName}).Inc() case <-timeout: h.logger.Debugw("Timed out waiting for messages to read.", zap.Duration("waited", h.readTimeout), zap.Int("read", len(msgs))) - return msgs, nil + break loop } } - httpSourceReadCount.With(map[string]string{metricspkg.LabelVertex: h.name, metricspkg.LabelPipeline: h.pipelineName}).Inc() h.logger.Debugf("Read %d messages.", len(msgs)) + if len(msgs) > 0 && !latest.IsZero() { + h.sourcePublishWM.PublishWatermark(processor.Watermark(latest), msgs[len(msgs)-1].ReadOffset) + } return msgs, nil } @@ -198,6 +230,7 @@ func (h *httpSource) Ack(_ context.Context, offsets []isb.Offset) []error { func (h *httpSource) Close() error { h.logger.Info("Shutting down http source server...") + h.cancelfn() close(h.messages) if err := h.shutdown(context.Background()); err != nil { return err diff --git a/pkg/sources/http/http_test.go b/pkg/sources/http/http_test.go index 33d8058db7..d11aa0ba23 100644 --- a/pkg/sources/http/http_test.go +++ b/pkg/sources/http/http_test.go @@ -7,6 +7,8 @@ import ( dfv1 "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1" "github.com/numaproj/numaflow/pkg/isb" "github.com/numaproj/numaflow/pkg/isb/stores/simplebuffer" + "github.com/numaproj/numaflow/pkg/watermark/generic" + "github.com/numaproj/numaflow/pkg/watermark/store/noop" "github.com/stretchr/testify/assert" ) @@ -39,8 +41,15 @@ func Test_NewHTTP(t *testing.T) { }, }, } + vi := &dfv1.VertexInstance{ + Vertex: v, + Hostname: "test-host", + Replica: 0, + } dest := []isb.BufferWriter{simplebuffer.NewInMemoryBuffer("test", 100)} - h, err := New(v, dest) + publishWMStore := generic.BuildPublishWMStores(noop.NewKVNoOpStore(), noop.NewKVNoOpStore()) + fetchWatermark, publishWatermark := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string]isb.BufferWriter{}) + h, err := New(vi, dest, fetchWatermark, publishWatermark, publishWMStore) assert.NoError(t, err) assert.False(t, h.ready) assert.Equal(t, v.Spec.Name, h.GetName()) diff --git a/pkg/sources/kafka/handler_test.go b/pkg/sources/kafka/handler_test.go index 7604a039ac..181acef881 100644 --- a/pkg/sources/kafka/handler_test.go +++ b/pkg/sources/kafka/handler_test.go @@ -11,6 +11,8 @@ import ( "github.com/numaproj/numaflow/pkg/isb" "github.com/numaproj/numaflow/pkg/isb/stores/simplebuffer" "github.com/numaproj/numaflow/pkg/shared/logging" + "github.com/numaproj/numaflow/pkg/watermark/generic" + "github.com/numaproj/numaflow/pkg/watermark/store/noop" "github.com/stretchr/testify/assert" ) @@ -37,8 +39,14 @@ func TestMessageHandling(t *testing.T) { }, }, }} - - ks, _ := NewKafkaSource(vertex, dest, WithLogger(logging.NewLogger()), + vi := &dfv1.VertexInstance{ + Vertex: vertex, + Hostname: "test-host", + Replica: 0, + } + publishWMStore := generic.BuildPublishWMStores(noop.NewKVNoOpStore(), noop.NewKVNoOpStore()) + fetchWatermark, publishWatermark := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string]isb.BufferWriter{}) + ks, _ := NewKafkaSource(vi, dest, fetchWatermark, publishWatermark, publishWMStore, WithLogger(logging.NewLogger()), WithBufferSize(100), WithReadTimeOut(100*time.Millisecond)) msg := &sarama.ConsumerMessage{ diff --git a/pkg/sources/kafka/reader.go b/pkg/sources/kafka/reader.go index 98400d310c..530bbc92f5 100644 --- a/pkg/sources/kafka/reader.go +++ b/pkg/sources/kafka/reader.go @@ -17,8 +17,12 @@ import ( "github.com/numaproj/numaflow/pkg/isb/forward" metricspkg "github.com/numaproj/numaflow/pkg/metrics" "github.com/numaproj/numaflow/pkg/shared/logging" - "github.com/numaproj/numaflow/pkg/shared/util" + sharedutil "github.com/numaproj/numaflow/pkg/shared/util" "github.com/numaproj/numaflow/pkg/udf/applier" + "github.com/numaproj/numaflow/pkg/watermark/fetch" + "github.com/numaproj/numaflow/pkg/watermark/generic" + "github.com/numaproj/numaflow/pkg/watermark/processor" + "github.com/numaproj/numaflow/pkg/watermark/publish" ) type KafkaSource struct { @@ -54,6 +58,11 @@ type KafkaSource struct { adminClient sarama.ClusterAdmin // sarama client saramaClient sarama.Client + // source watermark publishers for different partitions + sourcePublishWMs map[int32]publish.Publisher + // source watermark publisher stores + srcPublishWMStores *generic.PublishWMStores + lock *sync.RWMutex } type Option func(*KafkaSource) error @@ -100,24 +109,48 @@ func (r *KafkaSource) GetName() string { // There is a chance that we have read the message and the container got forcefully terminated before processing. To provide // at-least-once semantics for reading, during restart we will have to reprocess all unacknowledged messages. func (r *KafkaSource) Read(_ context.Context, count int64) ([]*isb.ReadMessage, error) { + // It stores latest timestamps for different partitions + latestTimestamps := make(map[int32]time.Time) msgs := make([]*isb.ReadMessage, 0, count) - var i int64 - for i = 0; i < count; i++ { + timeout := time.After(r.readTimeout) +loop: + for i := int64(0); i < count; i++ { select { case m := <-r.handler.messages: kafkaSourceReadCount.With(map[string]string{metricspkg.LabelVertex: r.name, metricspkg.LabelPipeline: r.pipelineName}).Inc() - msgs = append(msgs, toReadMessage(m)) - - case <-time.After(r.readTimeout): + _m := toReadMessage(m) + msgs = append(msgs, _m) + // Get latest timestamps for different partitions + if t, ok := latestTimestamps[m.Partition]; !ok || m.Timestamp.After(t) { + latestTimestamps[m.Partition] = m.Timestamp + } + case <-timeout: // log that timeout has happened and don't return an error r.logger.Debugw("Timed out waiting for messages to read.", zap.Duration("waited", r.readTimeout)) - return msgs, nil + break loop } } - + for p, t := range latestTimestamps { + publisher := r.loadSourceWartermarkPublisher(p) + publisher.PublishWatermark(processor.Watermark(t), nil) // Source publisher does not care about the offset + } return msgs, nil } +// loadSourceWartermarkPublisher does a lazy load on the wartermark publisher +func (r *KafkaSource) loadSourceWartermarkPublisher(partitionID int32) publish.Publisher { + r.lock.Lock() + defer r.lock.Unlock() + if p, ok := r.sourcePublishWMs[partitionID]; ok { + return p + } + entityName := fmt.Sprintf("%s-%s-%d", r.pipelineName, r.name, partitionID) + processorEntity := processor.NewProcessorEntity(entityName) + sourcePublishWM := publish.NewPublish(r.lifecyclectx, processorEntity, r.srcPublishWMStores.HBStore, r.srcPublishWMStores.OTStore, publish.IsSource(), publish.WithDelay(sharedutil.GetWatermarkMaxDelay())) + r.sourcePublishWMs[partitionID] = sourcePublishWM + return sourcePublishWM +} + // Ack acknowledges an array of offset. func (r *KafkaSource) Ack(_ context.Context, offsets []isb.Offset) []error { // we want to block the handler from exiting if there are any inflight acks. @@ -206,16 +239,19 @@ func (r *KafkaSource) Pending(ctx context.Context) (int64, error) { } // NewKafkaSource returns a KafkaSource reader based on Kafka Consumer Group . -func NewKafkaSource(vertex *dfv1.Vertex, writers []isb.BufferWriter, opts ...Option) (*KafkaSource, error) { - source := vertex.Spec.Source.Kafka +func NewKafkaSource(vertexInstance *dfv1.VertexInstance, writers []isb.BufferWriter, fetchWM fetch.Fetcher, publishWM map[string]publish.Publisher, publishWMStores *generic.PublishWMStores, opts ...Option) (*KafkaSource, error) { + source := vertexInstance.Vertex.Spec.Source.Kafka kafkasource := &KafkaSource{ - name: vertex.Spec.Name, - pipelineName: vertex.Spec.PipelineName, - topic: source.Topic, - brokers: source.Brokers, - readTimeout: 1 * time.Second, // default timeout - handlerbuffer: 100, // default buffer size for kafka reads - logger: logging.NewLogger(), // default logger + name: vertexInstance.Vertex.Spec.Name, + pipelineName: vertexInstance.Vertex.Spec.PipelineName, + topic: source.Topic, + brokers: source.Brokers, + readTimeout: 1 * time.Second, // default timeout + handlerbuffer: 100, // default buffer size for kafka reads + srcPublishWMStores: publishWMStores, + sourcePublishWMs: make(map[int32]publish.Publisher, 0), + lock: new(sync.RWMutex), + logger: logging.NewLogger(), // default logger } for _, o := range opts { @@ -232,7 +268,7 @@ func NewKafkaSource(vertex *dfv1.Vertex, writers []isb.BufferWriter, opts ...Opt if t := source.TLS; t != nil { config.Net.TLS.Enable = true - if c, err := util.GetTLSConfig(t); err != nil { + if c, err := sharedutil.GetTLSConfig(t); err != nil { return nil, err } else { config.Net.TLS.Config = c @@ -267,13 +303,13 @@ func NewKafkaSource(vertex *dfv1.Vertex, writers []isb.BufferWriter, opts ...Opt destinations[w.GetName()] = w } - forwardOpts := []forward.Option{forward.WithLogger(kafkasource.logger)} - if x := vertex.Spec.Limits; x != nil { + forwardOpts := []forward.Option{forward.FromSourceVertex(), forward.WithLogger(kafkasource.logger)} + if x := vertexInstance.Vertex.Spec.Limits; x != nil { if x.ReadBatchSize != nil { forwardOpts = append(forwardOpts, forward.WithReadBatchSize(int64(*x.ReadBatchSize))) } } - forwarder, err := forward.NewInterStepDataForward(vertex, kafkasource, destinations, forward.All, applier.Terminal, nil, nil, forwardOpts...) + forwarder, err := forward.NewInterStepDataForward(vertexInstance.Vertex, kafkasource, destinations, forward.All, applier.Terminal, fetchWM, publishWM, forwardOpts...) if err != nil { kafkasource.logger.Errorw("Error instantiating the forwarder", zap.Error(err)) return nil, err @@ -283,7 +319,7 @@ func NewKafkaSource(vertex *dfv1.Vertex, writers []isb.BufferWriter, opts ...Opt } func configFromOpts(yamlconfig string) (*sarama.Config, error) { - config, err := util.GetSaramaConfigFromYAMLString(yamlconfig) + config, err := sharedutil.GetSaramaConfigFromYAMLString(yamlconfig) if err != nil { return nil, err } diff --git a/pkg/sources/kafka/reader_test.go b/pkg/sources/kafka/reader_test.go index 291a741509..824ad33f4a 100644 --- a/pkg/sources/kafka/reader_test.go +++ b/pkg/sources/kafka/reader_test.go @@ -9,6 +9,8 @@ import ( "github.com/numaproj/numaflow/pkg/isb" "github.com/numaproj/numaflow/pkg/isb/stores/simplebuffer" "github.com/numaproj/numaflow/pkg/shared/logging" + "github.com/numaproj/numaflow/pkg/watermark/generic" + "github.com/numaproj/numaflow/pkg/watermark/store/noop" "github.com/stretchr/testify/assert" ) @@ -25,7 +27,14 @@ func TestNewKafkasource(t *testing.T) { }, }, }} - ks, err := NewKafkaSource(vertex, dest, WithLogger(logging.NewLogger()), WithBufferSize(100), WithReadTimeOut(100*time.Millisecond), WithGroupName("default")) + vi := &dfv1.VertexInstance{ + Vertex: vertex, + Hostname: "test-host", + Replica: 0, + } + publishWMStore := generic.BuildPublishWMStores(noop.NewKVNoOpStore(), noop.NewKVNoOpStore()) + fetchWatermark, publishWatermark := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string]isb.BufferWriter{}) + ks, err := NewKafkaSource(vi, dest, fetchWatermark, publishWatermark, publishWMStore, WithLogger(logging.NewLogger()), WithBufferSize(100), WithReadTimeOut(100*time.Millisecond), WithGroupName("default")) // no errors if everything is good. assert.Nil(t, err) @@ -54,7 +63,14 @@ func TestGroupNameOverride(t *testing.T) { }, }, }} - ks, _ := NewKafkaSource(vertex, dest, WithLogger(logging.NewLogger()), WithBufferSize(100), WithReadTimeOut(100*time.Millisecond), WithGroupName("default")) + vi := &dfv1.VertexInstance{ + Vertex: vertex, + Hostname: "test-host", + Replica: 0, + } + publishWMStore := generic.BuildPublishWMStores(noop.NewKVNoOpStore(), noop.NewKVNoOpStore()) + fetchWatermark, publishWatermark := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string]isb.BufferWriter{}) + ks, _ := NewKafkaSource(vi, dest, fetchWatermark, publishWatermark, publishWMStore, WithLogger(logging.NewLogger()), WithBufferSize(100), WithReadTimeOut(100*time.Millisecond), WithGroupName("default")) assert.Equal(t, "default", ks.groupName) @@ -73,7 +89,14 @@ func TestDefaultBufferSize(t *testing.T) { }, }, }} - ks, _ := NewKafkaSource(vertex, dest, WithLogger(logging.NewLogger()), WithReadTimeOut(100*time.Millisecond), WithGroupName("default")) + vi := &dfv1.VertexInstance{ + Vertex: vertex, + Hostname: "test-host", + Replica: 0, + } + publishWMStore := generic.BuildPublishWMStores(noop.NewKVNoOpStore(), noop.NewKVNoOpStore()) + fetchWatermark, publishWatermark := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string]isb.BufferWriter{}) + ks, _ := NewKafkaSource(vi, dest, fetchWatermark, publishWatermark, publishWMStore, WithLogger(logging.NewLogger()), WithReadTimeOut(100*time.Millisecond), WithGroupName("default")) assert.Equal(t, 100, ks.handlerbuffer) @@ -92,7 +115,14 @@ func TestBufferSizeOverrides(t *testing.T) { }, }, }} - ks, _ := NewKafkaSource(vertex, dest, WithLogger(logging.NewLogger()), WithBufferSize(110), WithReadTimeOut(100*time.Millisecond), WithGroupName("default")) + vi := &dfv1.VertexInstance{ + Vertex: vertex, + Hostname: "test-host", + Replica: 0, + } + publishWMStore := generic.BuildPublishWMStores(noop.NewKVNoOpStore(), noop.NewKVNoOpStore()) + fetchWatermark, publishWatermark := generic.BuildNoOpWatermarkProgressorsFromBufferMap(map[string]isb.BufferWriter{}) + ks, _ := NewKafkaSource(vi, dest, fetchWatermark, publishWatermark, publishWMStore, WithLogger(logging.NewLogger()), WithBufferSize(110), WithReadTimeOut(100*time.Millisecond), WithGroupName("default")) assert.Equal(t, 110, ks.handlerbuffer) @@ -107,7 +137,7 @@ func TestOffsetFrom(t *testing.T) { assert.Equal(t, int64(64), offset) } -func TestToOffset(t *testing.T) { +func TestOffset(t *testing.T) { topic := "t1" partition := int32(1) offset := int64(23) diff --git a/pkg/sources/source.go b/pkg/sources/source.go index 1950412443..ac3c0f19de 100644 --- a/pkg/sources/source.go +++ b/pkg/sources/source.go @@ -148,9 +148,9 @@ func (sp *SourceProcessor) getSourcer(writers []isb.BufferWriter, fetchWM fetch. if l := sp.VertexInstance.Vertex.Spec.Limits; l != nil && l.ReadTimeout != nil { readOptions = append(readOptions, kafka.WithReadTimeOut(l.ReadTimeout.Duration)) } - return kafka.NewKafkaSource(sp.VertexInstance.Vertex, writers, readOptions...) + return kafka.NewKafkaSource(sp.VertexInstance, writers, fetchWM, publishWM, publishWMStores, readOptions...) } else if x := src.HTTP; x != nil { - return http.New(sp.VertexInstance.Vertex, writers, http.WithLogger(logger)) + return http.New(sp.VertexInstance, writers, fetchWM, publishWM, publishWMStores, http.WithLogger(logger)) } return nil, fmt.Errorf("invalid source spec") } diff --git a/pkg/watermark/fetch/edge.go b/pkg/watermark/fetch/edge_fetcher.go similarity index 65% rename from pkg/watermark/fetch/edge.go rename to pkg/watermark/fetch/edge_fetcher.go index b893a5d2ca..425250b9dd 100644 --- a/pkg/watermark/fetch/edge.go +++ b/pkg/watermark/fetch/edge_fetcher.go @@ -14,43 +14,38 @@ import ( "github.com/numaproj/numaflow/pkg/watermark/processor" ) -// Fetcher fetches watermark data from Vn-1 vertex. -type Fetcher interface { - // GetWatermark returns the inorder monotonically increasing watermark of the edge connected to Vn-1. - GetWatermark(offset isb.Offset) processor.Watermark - // GetHeadWatermark returns the latest watermark based on the head offset - GetHeadWatermark() processor.Watermark +// edgeFetcher is a fetcher between two vertices. +type edgeFetcher struct { + ctx context.Context + edgeName string + processorManager *ProcessorManager + log *zap.SugaredLogger } -// Edge is the edge relation between two vertices. -type Edge struct { - ctx context.Context - edgeName string - fromVertex FromVertexer - log *zap.SugaredLogger -} - -// NewEdgeBuffer returns a new Edge. FromVertex has the details about the processors responsible for writing to this +// NewEdgeFetcher returns a new edge fetcher, processorManager has the details about the processors responsible for writing to this // edge. -func NewEdgeBuffer(ctx context.Context, edgeName string, fromV FromVertexer) *Edge { - return &Edge{ - ctx: ctx, - edgeName: edgeName, - fromVertex: fromV, - log: logging.FromContext(ctx).With("edgeName", edgeName), +func NewEdgeFetcher(ctx context.Context, edgeName string, processorManager *ProcessorManager) Fetcher { + return &edgeFetcher{ + ctx: ctx, + edgeName: edgeName, + processorManager: processorManager, + log: logging.FromContext(ctx).With("edgeName", edgeName), } } // GetHeadWatermark returns the watermark using the HeadOffset (latest offset). This // can be used in showing the watermark progression for a vertex when not consuming the messages // directly (eg. UX, tests,) -func (e *Edge) GetHeadWatermark() processor.Watermark { +func (e *edgeFetcher) GetHeadWatermark() processor.Watermark { var debugString strings.Builder var headOffset int64 = math.MinInt64 var epoch int64 = math.MaxInt64 - var allProcessors = e.fromVertex.GetAllProcessors() + var allProcessors = e.processorManager.GetAllProcessors() // get the head offset of each processor for _, p := range allProcessors { + if !p.IsActive() { + continue + } e.log.Debugf("Processor: %v (headoffset:%d)", p, p.offsetTimeline.GetHeadOffset()) debugString.WriteString(fmt.Sprintf("[Processor:%v] (headoffset:%d) \n", p, p.offsetTimeline.GetHeadOffset())) var o = p.offsetTimeline.GetHeadOffset() @@ -68,7 +63,7 @@ func (e *Edge) GetHeadWatermark() processor.Watermark { } // GetWatermark gets the smallest timestamp for the given offset -func (e *Edge) GetWatermark(inputOffset isb.Offset) processor.Watermark { +func (e *edgeFetcher) GetWatermark(inputOffset isb.Offset) processor.Watermark { var offset, err = inputOffset.Sequence() if err != nil { e.log.Errorw("unable to get offset from isb.Offset.Sequence()", zap.Error(err)) @@ -76,7 +71,7 @@ func (e *Edge) GetWatermark(inputOffset isb.Offset) processor.Watermark { } var debugString strings.Builder var epoch int64 = math.MaxInt64 - var allProcessors = e.fromVertex.GetAllProcessors() + var allProcessors = e.processorManager.GetAllProcessors() for _, p := range allProcessors { if !p.IsActive() { continue diff --git a/pkg/watermark/fetch/edge_test.go b/pkg/watermark/fetch/edge_fetcher_test.go similarity index 70% rename from pkg/watermark/fetch/edge_test.go rename to pkg/watermark/fetch/edge_fetcher_test.go index ad7b4b826c..4917dc194c 100644 --- a/pkg/watermark/fetch/edge_test.go +++ b/pkg/watermark/fetch/edge_fetcher_test.go @@ -50,7 +50,7 @@ func TestBuffer_GetWatermark(t *testing.T) { hbWatcher, err := jetstream.NewKVJetStreamKVWatch(ctx, "testFetch", publisherHBBucketName, defaultJetStreamClient) otWatcher, err := jetstream.NewKVJetStreamKVWatch(ctx, "testFetch", publisherOTBucketName, defaultJetStreamClient) - testVertex := NewFromVertex(ctx, hbWatcher, otWatcher).(*fromVertex) + processorManager := NewProcessorManager(ctx, hbWatcher, otWatcher) var ( // TODO: watcher should not be nil testPod0 = NewProcessorToFetch(ctx, processor.NewProcessorEntity("testPod1"), 5, otWatcher) @@ -86,70 +86,70 @@ func TestBuffer_GetWatermark(t *testing.T) { for _, watermark := range pod2Timeline { testPod2.offsetTimeline.Put(watermark) } - testVertex.addProcessor("testPod0", testPod0) - testVertex.addProcessor("testPod1", testPod1) - testVertex.addProcessor("testPod2", testPod2) + processorManager.addProcessor("testPod0", testPod0) + processorManager.addProcessor("testPod1", testPod1) + processorManager.addProcessor("testPod2", testPod2) type args struct { offset int64 } tests := []struct { - name string - fromVertex FromVertexer - args args - want int64 + name string + processorManager *ProcessorManager + args args + want int64 }{ { - name: "offset_9", - fromVertex: testVertex, - args: args{9}, - want: time.Time{}.Unix(), + name: "offset_9", + processorManager: processorManager, + args: args{9}, + want: time.Time{}.Unix(), }, { - name: "offset_15", - fromVertex: testVertex, - args: args{15}, - want: 8, + name: "offset_15", + processorManager: processorManager, + args: args{15}, + want: 8, }, { - name: "offset_18", - fromVertex: testVertex, - args: args{18}, - want: 9, + name: "offset_18", + processorManager: processorManager, + args: args{18}, + want: 9, }, { - name: "offset_22", - fromVertex: testVertex, - args: args{22}, - want: 10, + name: "offset_22", + processorManager: processorManager, + args: args{22}, + want: 10, }, { - name: "offset_22", - fromVertex: testVertex, - args: args{23}, - want: 10, + name: "offset_22", + processorManager: processorManager, + args: args{23}, + want: 10, }, { - name: "offset_28", - fromVertex: testVertex, - args: args{28}, - want: 14, + name: "offset_28", + processorManager: processorManager, + args: args{28}, + want: 14, }, { - name: "offset_28", - fromVertex: testVertex, - args: args{29}, - want: 17, + name: "offset_28", + processorManager: processorManager, + args: args{29}, + want: 17, }, } location, _ := time.LoadLocation("UTC") for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - b := &Edge{ - ctx: ctx, - edgeName: "testBuffer", - fromVertex: tt.fromVertex, - log: zaptest.NewLogger(t).Sugar(), + b := &edgeFetcher{ + ctx: ctx, + edgeName: "testBuffer", + processorManager: tt.processorManager, + log: zaptest.NewLogger(t).Sugar(), } if got := b.GetWatermark(isb.SimpleOffset(func() string { return strconv.FormatInt(tt.args.offset, 10) })); time.Time(got).In(location) != time.Unix(tt.want, 0).In(location) { t.Errorf("GetWatermark() = %v, want %v", got, processor.Watermark(time.Unix(tt.want, 0))) diff --git a/pkg/watermark/fetch/interface.go b/pkg/watermark/fetch/interface.go new file mode 100644 index 0000000000..38599a0e47 --- /dev/null +++ b/pkg/watermark/fetch/interface.go @@ -0,0 +1,14 @@ +package fetch + +import ( + "github.com/numaproj/numaflow/pkg/isb" + "github.com/numaproj/numaflow/pkg/watermark/processor" +) + +// Fetcher fetches watermark data from Vn-1 vertex. +type Fetcher interface { + // GetWatermark returns the inorder monotonically increasing watermark of the edge connected to Vn-1. + GetWatermark(offset isb.Offset) processor.Watermark + // GetHeadWatermark returns the latest watermark based on the head offset + GetHeadWatermark() processor.Watermark +} diff --git a/pkg/watermark/fetch/offset_timeline.go b/pkg/watermark/fetch/offset_timeline.go index 2d7339ca7d..235eccb9b6 100644 --- a/pkg/watermark/fetch/offset_timeline.go +++ b/pkg/watermark/fetch/offset_timeline.go @@ -106,6 +106,16 @@ func (t *OffsetTimeline) GetHeadOffset() int64 { return t.watermarks.Front().Value.(OffsetWatermark).offset } +// GetHeadWatermark returns the head watermark, which is the highest one. +func (t *OffsetTimeline) GetHeadWatermark() int64 { + t.lock.RLock() + defer t.lock.RUnlock() + if t.watermarks.Len() == 0 { + return 0 + } + return t.watermarks.Front().Value.(OffsetWatermark).watermark +} + // GetTailOffset returns the smallest offset with the smallest watermark. func (t *OffsetTimeline) GetTailOffset() int64 { t.lock.RLock() diff --git a/pkg/watermark/fetch/options.go b/pkg/watermark/fetch/options.go index bbd5723bcb..7557adbace 100644 --- a/pkg/watermark/fetch/options.go +++ b/pkg/watermark/fetch/options.go @@ -1,31 +1,31 @@ package fetch -type vertexOptions struct { +type processorManagerOptions struct { podHeartbeatRate int64 refreshingProcessorsRate int64 separateOTBucket bool } -// VertexOption set options for FromVertex. -type VertexOption func(*vertexOptions) +// ProcessorManagerOption set options for FromVertex. +type ProcessorManagerOption func(*processorManagerOptions) // WithPodHeartbeatRate sets the heartbeat rate in seconds. -func WithPodHeartbeatRate(rate int64) VertexOption { - return func(opts *vertexOptions) { +func WithPodHeartbeatRate(rate int64) ProcessorManagerOption { + return func(opts *processorManagerOptions) { opts.podHeartbeatRate = rate } } // WithRefreshingProcessorsRate sets the processor refreshing rate in seconds. -func WithRefreshingProcessorsRate(rate int64) VertexOption { - return func(opts *vertexOptions) { +func WithRefreshingProcessorsRate(rate int64) ProcessorManagerOption { + return func(opts *processorManagerOptions) { opts.refreshingProcessorsRate = rate } } // WithSeparateOTBuckets creates a different bucket for maintaining each processor offset-timeline. -func WithSeparateOTBuckets(separate bool) VertexOption { - return func(opts *vertexOptions) { +func WithSeparateOTBuckets(separate bool) ProcessorManagerOption { + return func(opts *processorManagerOptions) { opts.separateOTBucket = separate } } diff --git a/pkg/watermark/fetch/options_test.go b/pkg/watermark/fetch/options_test.go index bd75a776cf..c33fdea3b5 100644 --- a/pkg/watermark/fetch/options_test.go +++ b/pkg/watermark/fetch/options_test.go @@ -7,11 +7,11 @@ import ( ) func TestOptions(t *testing.T) { - testOpts := []VertexOption{ + testOpts := []ProcessorManagerOption{ WithPodHeartbeatRate(10), WithRefreshingProcessorsRate(15), } - opts := &vertexOptions{ + opts := &processorManagerOptions{ podHeartbeatRate: 5, refreshingProcessorsRate: 5, } diff --git a/pkg/watermark/fetch/vertex.go b/pkg/watermark/fetch/processor_manager.go similarity index 80% rename from pkg/watermark/fetch/vertex.go rename to pkg/watermark/fetch/processor_manager.go index d1128279e2..53e3853100 100644 --- a/pkg/watermark/fetch/vertex.go +++ b/pkg/watermark/fetch/processor_manager.go @@ -15,21 +15,10 @@ import ( "github.com/numaproj/numaflow/pkg/watermark/store" ) -// FromVertexer defines an interface which builds the view of Vn-th vertex from the point of view of Vn-th vertex. -type FromVertexer interface { - // GetAllProcessors fetches all the processors from Vn-1 vertex. processors could be pods or when the vertex is a - // source vertex, it could be partitions if the source is Kafka. - GetAllProcessors() map[string]*ProcessorToFetch - // GetProcessor gets a processor. - GetProcessor(processor string) *ProcessorToFetch - // DeleteProcessor deletes a processor. - DeleteProcessor(processor string) -} - -// fromVertex is the point of view of Vn-1 from Vn vertex. The code is running on Vn vertex. +// ProcessorManager manages the point of view of Vn-1 from Vn vertex processors (or source processor). The code is running on Vn vertex. // It has the mapping of all the processors which in turn has all the information about each processor // timelines. -type fromVertex struct { +type ProcessorManager struct { ctx context.Context hbWatcher store.WatermarkKVWatcher otWatcher store.WatermarkKVWatcher @@ -39,12 +28,12 @@ type fromVertex struct { log *zap.SugaredLogger // opts - opts *vertexOptions + opts *processorManagerOptions } -// NewFromVertex returns `FromVertex` -func NewFromVertex(ctx context.Context, hbWatcher store.WatermarkKVWatcher, otWatcher store.WatermarkKVWatcher, inputOpts ...VertexOption) FromVertexer { - opts := &vertexOptions{ +// NewProcessorManager returns a new ProcessorManager instance +func NewProcessorManager(ctx context.Context, hbWatcher store.WatermarkKVWatcher, otWatcher store.WatermarkKVWatcher, inputOpts ...ProcessorManagerOption) *ProcessorManager { + opts := &processorManagerOptions{ podHeartbeatRate: 5, refreshingProcessorsRate: 5, separateOTBucket: false, @@ -53,7 +42,7 @@ func NewFromVertex(ctx context.Context, hbWatcher store.WatermarkKVWatcher, otWa opt(opts) } - v := &fromVertex{ + v := &ProcessorManager{ ctx: ctx, hbWatcher: hbWatcher, otWatcher: otWatcher, @@ -71,14 +60,14 @@ func NewFromVertex(ctx context.Context, hbWatcher store.WatermarkKVWatcher, otWa } // addProcessor adds a new processor. -func (v *fromVertex) addProcessor(processor string, p *ProcessorToFetch) { +func (v *ProcessorManager) addProcessor(processor string, p *ProcessorToFetch) { v.lock.Lock() defer v.lock.Unlock() v.processors[processor] = p } // GetProcessor gets a processor. -func (v *fromVertex) GetProcessor(processor string) *ProcessorToFetch { +func (v *ProcessorManager) GetProcessor(processor string) *ProcessorToFetch { v.lock.RLock() defer v.lock.RUnlock() if p, ok := v.processors[processor]; ok { @@ -88,7 +77,7 @@ func (v *fromVertex) GetProcessor(processor string) *ProcessorToFetch { } // DeleteProcessor deletes a processor. -func (v *fromVertex) DeleteProcessor(processor string) { +func (v *ProcessorManager) DeleteProcessor(processor string) { v.lock.Lock() defer v.lock.Unlock() // we do not require this processor's reference anymore @@ -96,7 +85,7 @@ func (v *fromVertex) DeleteProcessor(processor string) { } // GetAllProcessors returns all the processors. -func (v *fromVertex) GetAllProcessors() map[string]*ProcessorToFetch { +func (v *ProcessorManager) GetAllProcessors() map[string]*ProcessorToFetch { v.lock.RLock() defer v.lock.RUnlock() var processors = make(map[string]*ProcessorToFetch, len(v.processors)) @@ -106,7 +95,7 @@ func (v *fromVertex) GetAllProcessors() map[string]*ProcessorToFetch { return processors } -func (v *fromVertex) startRefreshingProcessors() { +func (v *ProcessorManager) startRefreshingProcessors() { ticker := time.NewTicker(time.Duration(v.opts.refreshingProcessorsRate) * time.Second) defer ticker.Stop() v.log.Infow("Refreshing ActiveProcessors ticker started") @@ -121,7 +110,7 @@ func (v *fromVertex) startRefreshingProcessors() { } // refreshingProcessors keeps the v.ActivePods to be a map of live ActivePods -func (v *fromVertex) refreshingProcessors() { +func (v *ProcessorManager) refreshingProcessors() { var debugStr strings.Builder for pName, pTime := range v.heartbeat.GetAll() { p := v.GetProcessor(pName) @@ -146,7 +135,7 @@ func (v *fromVertex) refreshingProcessors() { // startHeatBeatWatcher starts the processor Heartbeat Watcher to listen to the processor bucket and update the processor // Heartbeat map. In the processor heartbeat bucket we have the structure key: processor-name, value: processor-heartbeat. -func (v *fromVertex) startHeatBeatWatcher() { +func (v *ProcessorManager) startHeatBeatWatcher() { watchCh := v.hbWatcher.Watch(v.ctx) for { select { diff --git a/pkg/watermark/fetch/vertex_test.go b/pkg/watermark/fetch/processor_manager_test.go similarity index 86% rename from pkg/watermark/fetch/vertex_test.go rename to pkg/watermark/fetch/processor_manager_test.go index 4fcfa86d05..ad491e6ce2 100644 --- a/pkg/watermark/fetch/vertex_test.go +++ b/pkg/watermark/fetch/processor_manager_test.go @@ -73,8 +73,8 @@ func TestFetcherWithSameOTBucket(t *testing.T) { hbWatcher, err := jetstream.NewKVJetStreamKVWatch(ctx, "testFetch", keyspace+"_PROCESSORS", defaultJetStreamClient) otWatcher, err := jetstream.NewKVJetStreamKVWatch(ctx, "testFetch", keyspace+"_OT", defaultJetStreamClient) - var testVertex = NewFromVertex(ctx, hbWatcher, otWatcher, WithPodHeartbeatRate(1), WithRefreshingProcessorsRate(1), WithSeparateOTBuckets(false)).(*fromVertex) - var testBuffer = NewEdgeBuffer(ctx, "testBuffer", testVertex) + var testVertex = NewProcessorManager(ctx, hbWatcher, otWatcher, WithPodHeartbeatRate(1), WithRefreshingProcessorsRate(1), WithSeparateOTBuckets(false)) + var testBuffer = NewEdgeFetcher(ctx, "testBuffer", testVertex).(*edgeFetcher) go func() { var err error @@ -99,14 +99,14 @@ func TestFetcherWithSameOTBucket(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - allProcessors := testBuffer.fromVertex.GetAllProcessors() + allProcessors := testBuffer.processorManager.GetAllProcessors() for len(allProcessors) != 2 { select { case <-ctx.Done(): t.Fatalf("expected 2 processors, got %d: %s", len(allProcessors), ctx.Err()) default: time.Sleep(1 * time.Millisecond) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() } } @@ -120,24 +120,24 @@ func TestFetcherWithSameOTBucket(t *testing.T) { t.Fatalf("expected p1 to be deleted: %s", ctx.Err()) default: time.Sleep(1 * time.Millisecond) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() } } - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() assert.Equal(t, 2, len(allProcessors)) assert.True(t, allProcessors["p1"].IsDeleted()) assert.True(t, allProcessors["p2"].IsActive()) _ = testBuffer.GetWatermark(isb.SimpleOffset(func() string { return strconv.FormatInt(testOffset, 10) })) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() assert.Equal(t, 2, len(allProcessors)) assert.True(t, allProcessors["p1"].IsDeleted()) assert.True(t, allProcessors["p2"].IsActive()) // "p1" should be deleted after this GetWatermark offset=101 // because "p1" offsetTimeline's head offset=100, which is < inputOffset 103 _ = testBuffer.GetWatermark(isb.SimpleOffset(func() string { return strconv.FormatInt(testOffset+3, 10) })) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() assert.Equal(t, 2, len(allProcessors)) assert.True(t, allProcessors["p2"].IsActive()) assert.False(t, allProcessors["p1"].IsActive()) @@ -155,14 +155,14 @@ func TestFetcherWithSameOTBucket(t *testing.T) { // wait until p1 becomes active time.Sleep(time.Duration(testVertex.opts.podHeartbeatRate) * time.Second) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() for len(allProcessors) != 2 { select { case <-ctx.Done(): t.Fatalf("expected 2 processors, got %d: %s", len(allProcessors), ctx.Err()) default: time.Sleep(1 * time.Millisecond) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() } } @@ -171,7 +171,7 @@ func TestFetcherWithSameOTBucket(t *testing.T) { // "p1" has been deleted from vertex.Processors // so "p1" will be considered as a new processors and a new offsetTimeline watcher for "p1" will be created _ = testBuffer.GetWatermark(isb.SimpleOffset(func() string { return strconv.FormatInt(testOffset+1, 10) })) - newP1 := testBuffer.fromVertex.GetProcessor("p1") + newP1 := testBuffer.processorManager.GetProcessor("p1") assert.NotNil(t, newP1) assert.True(t, newP1.IsActive()) assert.NotNil(t, newP1.offsetTimeline) @@ -202,7 +202,7 @@ func TestFetcherWithSameOTBucket(t *testing.T) { t.Fatalf("expected p1 to be inactive: %s", ctx.Err()) default: time.Sleep(1 * time.Millisecond) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() } } @@ -218,14 +218,14 @@ func TestFetcherWithSameOTBucket(t *testing.T) { } }() - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() for len(allProcessors) != 2 { select { case <-ctx.Done(): t.Fatalf("expected 2 processors, got %d: %s", len(allProcessors), ctx.Err()) default: time.Sleep(1 * time.Millisecond) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() } } @@ -308,8 +308,8 @@ func TestFetcherWithSeparateOTBucket(t *testing.T) { hbWatcher, err := jetstream.NewKVJetStreamKVWatch(ctx, "testFetch", keyspace+"_PROCESSORS", defaultJetStreamClient) otWatcher, err := jetstream.NewKVJetStreamKVWatch(ctx, "testFetch", keyspace+"_OT", defaultJetStreamClient) - var testVertex = NewFromVertex(ctx, hbWatcher, otWatcher, WithPodHeartbeatRate(1), WithRefreshingProcessorsRate(1), WithSeparateOTBuckets(true)).(*fromVertex) - var testBuffer = NewEdgeBuffer(ctx, "testBuffer", testVertex) + var testVertex = NewProcessorManager(ctx, hbWatcher, otWatcher, WithPodHeartbeatRate(1), WithRefreshingProcessorsRate(1), WithSeparateOTBuckets(true)) + var testBuffer = NewEdgeFetcher(ctx, "testBuffer", testVertex).(*edgeFetcher) //var location, _ = time.LoadLocation("UTC") go func() { @@ -334,14 +334,14 @@ func TestFetcherWithSeparateOTBucket(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - allProcessors := testBuffer.fromVertex.GetAllProcessors() + allProcessors := testBuffer.processorManager.GetAllProcessors() for len(allProcessors) != 2 { select { case <-ctx.Done(): t.Fatalf("expected 2 processors, got %d: %s", len(allProcessors), ctx.Err()) default: time.Sleep(1 * time.Millisecond) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() } } @@ -355,23 +355,23 @@ func TestFetcherWithSeparateOTBucket(t *testing.T) { t.Fatalf("expected p1 to be deleted: %s", ctx.Err()) default: time.Sleep(1 * time.Millisecond) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() } } - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() assert.Equal(t, 2, len(allProcessors)) assert.True(t, allProcessors["p1"].IsDeleted()) assert.True(t, allProcessors["p2"].IsActive()) _ = testBuffer.GetWatermark(isb.SimpleOffset(func() string { return strconv.FormatInt(testOffset, 10) })) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() assert.Equal(t, 2, len(allProcessors)) assert.True(t, allProcessors["p1"].IsDeleted()) assert.True(t, allProcessors["p2"].IsActive()) // "p1" should be deleted after this GetWatermark offset=101 // because "p1" offsetTimeline's head offset=100, which is < inputOffset 103 _ = testBuffer.GetWatermark(isb.SimpleOffset(func() string { return strconv.FormatInt(testOffset+3, 10) })) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() assert.Equal(t, 1, len(allProcessors)) assert.True(t, allProcessors["p2"].IsActive()) @@ -388,14 +388,14 @@ func TestFetcherWithSeparateOTBucket(t *testing.T) { }() // wait until p1 becomes active - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() for len(allProcessors) != 2 { select { case <-ctx.Done(): t.Fatalf("expected 2 processors, got %d: %s", len(allProcessors), ctx.Err()) default: time.Sleep(1 * time.Millisecond) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() } } @@ -405,7 +405,7 @@ func TestFetcherWithSeparateOTBucket(t *testing.T) { // "p1" has been deleted from vertex.Processors // so "p1" will be considered as a new processors and a new offsetTimeline watcher for "p1" will be created _ = testBuffer.GetWatermark(isb.SimpleOffset(func() string { return strconv.FormatInt(testOffset+1, 10) })) - newP1 := testBuffer.fromVertex.GetProcessor("p1") + newP1 := testBuffer.processorManager.GetProcessor("p1") assert.NotNil(t, newP1) assert.True(t, newP1.IsActive()) assert.NotNil(t, newP1.offsetTimeline) @@ -424,7 +424,7 @@ func TestFetcherWithSeparateOTBucket(t *testing.T) { t.Fatalf("expected p1 to be inactive: %s", ctx.Err()) default: time.Sleep(1 * time.Millisecond) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() } } @@ -440,14 +440,14 @@ func TestFetcherWithSeparateOTBucket(t *testing.T) { } }() - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() for len(allProcessors) != 2 { select { case <-ctx.Done(): t.Fatalf("expected 2 processors, got %d: %s", len(allProcessors), ctx.Err()) default: time.Sleep(1 * time.Millisecond) - allProcessors = testBuffer.fromVertex.GetAllProcessors() + allProcessors = testBuffer.processorManager.GetAllProcessors() } } diff --git a/pkg/watermark/fetch/source_fetcher.go b/pkg/watermark/fetch/source_fetcher.go new file mode 100644 index 0000000000..223370ed53 --- /dev/null +++ b/pkg/watermark/fetch/source_fetcher.go @@ -0,0 +1,66 @@ +package fetch + +import ( + "context" + "math" + "time" + + "github.com/numaproj/numaflow/pkg/isb" + "github.com/numaproj/numaflow/pkg/shared/logging" + "github.com/numaproj/numaflow/pkg/watermark/processor" + "go.uber.org/zap" +) + +// sourceFetcher is a fetcher on source buffers. +type sourceFetcher struct { + ctx context.Context + sourceBufferName string + processorManager *ProcessorManager + log *zap.SugaredLogger +} + +// NewSourceFetcher returns a new source fetcher, processorManager has the details about the processors responsible for writing to the +// buckets of the source buffer. +func NewSourceFetcher(ctx context.Context, sourceBufferName string, processorManager *ProcessorManager) Fetcher { + return &sourceFetcher{ + ctx: ctx, + sourceBufferName: sourceBufferName, + processorManager: processorManager, + log: logging.FromContext(ctx).With("sourceBufferName", sourceBufferName), + } +} + +// GetHeadWatermark returns the latest watermark of all the processors. +func (e *sourceFetcher) GetHeadWatermark() processor.Watermark { + var epoch int64 = math.MinInt64 + for _, p := range e.processorManager.GetAllProcessors() { + if !p.IsActive() { + continue + } + if p.offsetTimeline.GetHeadWatermark() > epoch { + epoch = p.offsetTimeline.GetHeadWatermark() + } + } + if epoch == math.MinInt64 { + return processor.Watermark(time.Time{}) + } + return processor.Watermark(time.Unix(epoch, 0)) +} + +// GetWatermark returns the lowest of the latest watermark of all the processors, +// it ignores the input offset. +func (e *sourceFetcher) GetWatermark(_ isb.Offset) processor.Watermark { + var epoch int64 = math.MaxInt64 + for _, p := range e.processorManager.GetAllProcessors() { + if !p.IsActive() { + continue + } + if p.offsetTimeline.GetHeadWatermark() < epoch { + epoch = p.offsetTimeline.GetHeadWatermark() + } + } + if epoch == math.MaxInt64 { + return processor.Watermark(time.Time{}) + } + return processor.Watermark(time.Unix(epoch, 0)) +} diff --git a/pkg/watermark/fetch/source_fetcher_test.go b/pkg/watermark/fetch/source_fetcher_test.go new file mode 100644 index 0000000000..fa442a6539 --- /dev/null +++ b/pkg/watermark/fetch/source_fetcher_test.go @@ -0,0 +1,5 @@ +//go:build isb_jetstream + +package fetch + +// TODO: Add test cases diff --git a/pkg/watermark/fetch/utils.go b/pkg/watermark/fetch/utils.go deleted file mode 100644 index 55c46e4c94..0000000000 --- a/pkg/watermark/fetch/utils.go +++ /dev/null @@ -1,38 +0,0 @@ -package fetch - -import ( - "time" - - "github.com/nats-io/nats.go" - "go.uber.org/zap" - - jsclient "github.com/numaproj/numaflow/pkg/shared/clients/jetstream" -) - -// _bucketWatchRetryCount is max number of retry to make sure we can connect to a bucket to watch. -const _bucketWatchRetryCount = 5 - -// _delayInSecBetweenBucketWatchRetry in seconds between bucket watch retry -const _delayInSecBetweenBucketWatchRetry = 1 - -// RetryUntilSuccessfulWatcherCreation creates a watcher and will wait till it is created if infiniteLoop is set to true. -// TODO: use `wait.ExponentialBackoffWithContext` -func RetryUntilSuccessfulWatcherCreation(js *jsclient.JetStreamContext, bucketName string, infiniteLoop bool, log *zap.SugaredLogger) nats.KeyWatcher { - for i := 0; i < _bucketWatchRetryCount || infiniteLoop; i++ { - bucket, err := js.KeyValue(bucketName) - if err != nil { - log.Errorw("Failed to get the bucket by bucket name", zap.String("bucket", bucketName), zap.Error(err)) - time.Sleep(_delayInSecBetweenBucketWatchRetry * time.Second) - continue - } - watcher, err := bucket.WatchAll() - if err != nil { - log.Errorw("Failed to create the watch all watcher for bucket name", zap.String("bucket", bucketName), zap.Error(err)) - time.Sleep(_delayInSecBetweenBucketWatchRetry * time.Second) - continue - } - log.Infow("Watcher created for bucket", zap.String("bucket", bucketName)) - return watcher - } - return nil -} diff --git a/pkg/watermark/fetch/utils_test.go b/pkg/watermark/fetch/utils_test.go deleted file mode 100644 index 413f5afa25..0000000000 --- a/pkg/watermark/fetch/utils_test.go +++ /dev/null @@ -1,43 +0,0 @@ -//go:build isb_jetstream - -package fetch - -import ( - "context" - "testing" - - "github.com/nats-io/nats.go" - "github.com/stretchr/testify/assert" - "go.uber.org/zap/zaptest" - - jsclient "github.com/numaproj/numaflow/pkg/shared/clients/jetstream" -) - -func TestRetryUntilSuccessfulWatcherCreation(t *testing.T) { - // Connect to NATS - nc, err := jsclient.NewDefaultJetStreamClient(nats.DefaultURL).Connect(context.TODO()) - assert.Nil(t, err) - - // Create JetStream Context - js, err := nc.JetStream(nats.PublishAsyncMaxPending(256)) - assert.Nil(t, err) - - js.CreateKeyValue(&nats.KeyValueConfig{ - Bucket: "utilTest", - Description: "", - MaxValueSize: 0, - History: 0, - TTL: 0, - MaxBytes: 0, - Storage: nats.MemoryStorage, - Replicas: 0, - Placement: nil, - }) - defer js.DeleteKeyValue("utilTest") - - watcher := RetryUntilSuccessfulWatcherCreation(js, "utilTest", false, zaptest.NewLogger(t).Sugar()) - assert.NotNil(t, watcher) - - watcherNil := RetryUntilSuccessfulWatcherCreation(js, "nonExist", false, zaptest.NewLogger(t).Sugar()) - assert.Nil(t, watcherNil) -} diff --git a/pkg/watermark/generic/fetcher.go b/pkg/watermark/generic/fetcher.go index 016e67e7d5..607175fa6a 100644 --- a/pkg/watermark/generic/fetcher.go +++ b/pkg/watermark/generic/fetcher.go @@ -3,19 +3,10 @@ package generic import ( "context" - "github.com/numaproj/numaflow/pkg/isb" "github.com/numaproj/numaflow/pkg/watermark/fetch" - "github.com/numaproj/numaflow/pkg/watermark/processor" "github.com/numaproj/numaflow/pkg/watermark/store" ) -// genericFetch is a generic fetcher which can be used for most use cases. -type genericFetch struct { - fromEdge *fetch.Edge -} - -var _ fetch.Fetcher = (*genericFetch)(nil) - // FetchWMWatchers has the watcher information required for fetching watermarks. type FetchWMWatchers struct { HBWatch store.WatermarkKVWatcher @@ -30,26 +21,16 @@ func BuildFetchWMWatchers(hbWatch store.WatermarkKVWatcher, otWatch store.Waterm } } -// NewGenericFetch returns GenericFetch. vertexName is the vertex currently processing. +// NewGenericEdgeFetch returns a Fetcher, where bufferName is the from buffer of the vertex that is currently processing. // fetchWM is a struct for retrieving both the heartbeat // and the offset watermark timeline (Vn-1 vertex). -func NewGenericFetch(ctx context.Context, vertexName string, fetchWM FetchWMWatchers) fetch.Fetcher { - fromVertex := fetch.NewFromVertex(ctx, fetchWM.HBWatch, fetchWM.OTWatch) - fromEdge := fetch.NewEdgeBuffer(ctx, vertexName, fromVertex) - - gf := &genericFetch{ - fromEdge: fromEdge, - } - - return gf -} - -// GetWatermark returns the watermark for the offset. -func (g *genericFetch) GetWatermark(offset isb.Offset) processor.Watermark { - return g.fromEdge.GetWatermark(offset) +func NewGenericEdgeFetch(ctx context.Context, bufferName string, fetchWM FetchWMWatchers) fetch.Fetcher { + processorManager := fetch.NewProcessorManager(ctx, fetchWM.HBWatch, fetchWM.OTWatch) + return fetch.NewEdgeFetcher(ctx, bufferName, processorManager) } -// GetHeadWatermark returns the head watermark based on the head offset. -func (g *genericFetch) GetHeadWatermark() processor.Watermark { - return g.fromEdge.GetHeadWatermark() +// NewGenericSourceFetch returns Fetcher, where sourceBufferName is the source buffer of the source vertex. +func NewGenericSourceFetch(ctx context.Context, sourceBufferName string, fetchWM FetchWMWatchers) fetch.Fetcher { + processorManager := fetch.NewProcessorManager(ctx, fetchWM.HBWatch, fetchWM.OTWatch) + return fetch.NewSourceFetcher(ctx, sourceBufferName, processorManager) } diff --git a/pkg/watermark/generic/jetstream/generic.go b/pkg/watermark/generic/jetstream/generic.go index 2679f291d7..6573787101 100644 --- a/pkg/watermark/generic/jetstream/generic.go +++ b/pkg/watermark/generic/jetstream/generic.go @@ -33,21 +33,26 @@ func BuildJetStreamWatermarkProgressors(ctx context.Context, vertexInstance *v1a publishWatermark := make(map[string]publish.Publisher) // Fetcher creation pipelineName := vertexInstance.Vertex.Spec.PipelineName - fromBufferName := vertexInstance.Vertex.GetFromBuffers()[0].Name - hbBucket := isbsvc.JetStreamProcessorBucket(pipelineName, fromBufferName) + fromBuffer := vertexInstance.Vertex.GetFromBuffers()[0] + hbBucket := isbsvc.JetStreamProcessorBucket(pipelineName, fromBuffer.Name) hbWatch, err := jetstream.NewKVJetStreamKVWatch(ctx, pipelineName, hbBucket, jsclient.NewInClusterJetStreamClient()) if err != nil { return nil, nil, fmt.Errorf("failed at new HB KVJetStreamKVWatch, HeartbeatBucket: %s, %w", hbBucket, err) } - otBucket := isbsvc.JetStreamOTBucket(pipelineName, fromBufferName) + otBucket := isbsvc.JetStreamOTBucket(pipelineName, fromBuffer.Name) otWatch, err := jetstream.NewKVJetStreamKVWatch(ctx, pipelineName, otBucket, jsclient.NewInClusterJetStreamClient()) if err != nil { return nil, nil, fmt.Errorf("failed at new OT KVJetStreamKVWatch, OTBucket: %s, %w", otBucket, err) } var fetchWmWatchers = generic.BuildFetchWMWatchers(hbWatch, otWatch) - fetchWatermark := generic.NewGenericFetch(ctx, vertexInstance.Vertex.Name, fetchWmWatchers) + var fetchWatermark fetch.Fetcher + if fromBuffer.Type == v1alpha1.SourceBuffer { + fetchWatermark = generic.NewGenericSourceFetch(ctx, fromBuffer.Name, fetchWmWatchers) + } else { + fetchWatermark = generic.NewGenericEdgeFetch(ctx, fromBuffer.Name, fetchWmWatchers) + } // Publisher map creation, we need a publisher per edge. diff --git a/pkg/watermark/generic/publisher.go b/pkg/watermark/generic/publisher.go index 1d6bbe5a8b..9fb359bbfb 100644 --- a/pkg/watermark/generic/publisher.go +++ b/pkg/watermark/generic/publisher.go @@ -3,7 +3,6 @@ package generic import ( "context" - "github.com/numaproj/numaflow/pkg/isb" "github.com/numaproj/numaflow/pkg/watermark/processor" "github.com/numaproj/numaflow/pkg/watermark/publish" "github.com/numaproj/numaflow/pkg/watermark/store" @@ -23,36 +22,9 @@ func BuildPublishWMStores(hbStore store.WatermarkKVStorer, otStore store.Waterma } } -// genericPublish is a generic publisher which will work for most cases. -type genericPublish struct { - toEdge publish.Publisher -} - -var _ publish.Publisher = (*genericPublish)(nil) - -// NewGenericPublish returns GenericPublish. processorName is the unique processor (pod) that is running on this vertex. -// publishKeyspace is obsolete, and will be removed in subsequent iterations. publishWM is a struct for storing both the heartbeat -// and the offset watermark timeline stores for the Vn vertex. +// NewGenericPublish returns GenericPublish, where processorName is the unique processor (pod) that is running on this +// vertex, publishWM is a struct for storing both the heartbeat and the offset watermark timeline stores for the Vn vertex. func NewGenericPublish(ctx context.Context, processorName string, publishWM PublishWMStores) publish.Publisher { publishEntity := processor.NewProcessorEntity(processorName) - udfPublish := publish.NewPublish(ctx, publishEntity, publishWM.HBStore, publishWM.OTStore) - gp := &genericPublish{ - toEdge: udfPublish, - } - return gp -} - -// PublishWatermark publishes for the generic publisher. -func (g *genericPublish) PublishWatermark(watermark processor.Watermark, offset isb.Offset) { - g.toEdge.PublishWatermark(watermark, offset) -} - -// GetLatestWatermark gets the latest watermakr for the generic publisher. -func (g *genericPublish) GetLatestWatermark() processor.Watermark { - return g.toEdge.GetLatestWatermark() -} - -// StopPublisher stops the generic publisher. -func (g *genericPublish) StopPublisher() { - g.toEdge.StopPublisher() + return publish.NewPublish(ctx, publishEntity, publishWM.HBStore, publishWM.OTStore) } diff --git a/pkg/watermark/processor/entity.go b/pkg/watermark/processor/entity.go index 5478800d70..7cf638adc0 100644 --- a/pkg/watermark/processor/entity.go +++ b/pkg/watermark/processor/entity.go @@ -19,7 +19,10 @@ func (w Watermark) String() string { var location, _ = time.LoadLocation("UTC") var t = time.Time(w).In(location) return t.Format(time.RFC3339) +} +func (w Watermark) After(t time.Time) bool { + return time.Time(w).After(t) } type entityOptions struct { diff --git a/pkg/watermark/publish/options.go b/pkg/watermark/publish/options.go index fc319fa0c3..41fd62eabb 100644 --- a/pkg/watermark/publish/options.go +++ b/pkg/watermark/publish/options.go @@ -1,11 +1,19 @@ package publish -import "github.com/nats-io/nats.go" +import ( + "time" +) type publishOptions struct { + // autoRefreshHeartbeat indicates whether to auto refresh heartbeat autoRefreshHeartbeat bool - bucketConfigs *nats.KeyValueConfig - podHeartbeatRate int64 + // The interval of refresh heartbeat + podHeartbeatRate int64 + // Watermark delay. + // It should only be used in a source publisher. + delay time.Duration + // Whether it is source publisher or not + isSource bool } type PublishOption func(*publishOptions) @@ -16,14 +24,22 @@ func WithAutoRefreshHeartbeatDisabled() PublishOption { } } -func WithBucketConfigs(cfgs *nats.KeyValueConfig) PublishOption { +func WithPodHeartbeatRate(rate int64) PublishOption { return func(opts *publishOptions) { - opts.bucketConfigs = cfgs + opts.podHeartbeatRate = rate } } -func WithPodHeartbeatRate(rate int64) PublishOption { +// WithDelay sets the watermark delay +func WithDelay(t time.Duration) PublishOption { return func(opts *publishOptions) { - opts.podHeartbeatRate = rate + opts.delay = t + } +} + +// IsSource indicates it's a source publisher +func IsSource() PublishOption { + return func(opts *publishOptions) { + opts.isSource = true } } diff --git a/pkg/watermark/publish/publisher.go b/pkg/watermark/publish/publisher.go index 8fc8980dab..8dd4fed72b 100644 --- a/pkg/watermark/publish/publisher.go +++ b/pkg/watermark/publish/publisher.go @@ -34,11 +34,7 @@ type publish struct { otStore store.WatermarkKVStorer log *zap.SugaredLogger headWatermark processor.Watermark - - // opts - // autoRefreshHeartbeat is not required for all processors. e.g. Kafka source doesn't need it - autoRefreshHeartbeat bool - podHeartbeatRate int64 + opts *publishOptions } // NewPublish returns `Publish`. @@ -49,25 +45,26 @@ func NewPublish(ctx context.Context, processorEntity processor.ProcessorEntitier opts := &publishOptions{ autoRefreshHeartbeat: true, podHeartbeatRate: 5, + isSource: false, + delay: 0, } for _, opt := range inputOpts { opt(opts) } p := &publish{ - ctx: ctx, - entity: processorEntity, - heartbeatStore: hbStore, - otStore: otStore, - log: log, - autoRefreshHeartbeat: opts.autoRefreshHeartbeat, - podHeartbeatRate: opts.podHeartbeatRate, + ctx: ctx, + entity: processorEntity, + heartbeatStore: hbStore, + otStore: otStore, + log: log, + opts: opts, } p.initialSetup() // TODO: i do not think we need this autoRefreshHeartbeat flag anymore. Remove it? - if p.autoRefreshHeartbeat { + if opts.autoRefreshHeartbeat { go p.publishHeartbeat() } @@ -82,11 +79,14 @@ func (p *publish) initialSetup() { // PublishWatermark publishes watermark and will retry until it can succeed. It will not publish if the new-watermark // is less than the current head watermark. func (p *publish) PublishWatermark(wm processor.Watermark, offset isb.Offset) { + if p.opts.isSource && p.opts.delay.Nanoseconds() > 0 && !time.Time(wm).IsZero() { + wm = processor.Watermark(time.Time(wm).Add(-p.opts.delay)) + } // update p.headWatermark only if wm > p.headWatermark if time.Time(wm).After(time.Time(p.headWatermark)) { p.headWatermark = wm } else { - p.log.Errorw("New watermark is older than the current watermark", zap.String("head", p.headWatermark.String()), zap.String("new", wm.String())) + p.log.Infow("New watermark is ignored because it's older than the current watermark", zap.String("head", p.headWatermark.String()), zap.String("new", wm.String())) return } @@ -95,8 +95,13 @@ func (p *publish) PublishWatermark(wm processor.Watermark, offset isb.Offset) { // build value (offset) value := make([]byte, 8) - o, _ := offset.Sequence() - binary.LittleEndian.PutUint64(value, uint64(o)) + var seq int64 + if p.opts.isSource { // For source publisher, we dont' care about the offset, also the sequence of the offset might not be integer. + seq = time.Now().UnixNano() + } else { + seq, _ = offset.Sequence() + } + binary.LittleEndian.PutUint64(value, uint64(seq)) for { err := p.otStore.PutKV(p.ctx, key, value) @@ -142,7 +147,7 @@ func (p *publish) GetLatestWatermark() processor.Watermark { } func (p *publish) publishHeartbeat() { - ticker := time.NewTicker(time.Second * time.Duration(p.podHeartbeatRate)) + ticker := time.NewTicker(time.Second * time.Duration(p.opts.podHeartbeatRate)) defer ticker.Stop() p.log.Infow("Refreshing ActiveProcessors ticker started") for { diff --git a/pkg/watermark/store/jetstream/kv_watch.go b/pkg/watermark/store/jetstream/kv_watch.go index 85c666564b..771872f7c5 100644 --- a/pkg/watermark/store/jetstream/kv_watch.go +++ b/pkg/watermark/store/jetstream/kv_watch.go @@ -110,7 +110,7 @@ func (k *jetStreamWatch) Watch(ctx context.Context) <-chan store.WatermarkKVEntr if value == nil { continue } - k.log.Info(value.Key(), value.Value(), value.Operation()) + k.log.Debug(value.Key(), value.Value(), value.Operation()) switch value.Operation() { case nats.KeyValuePut: updates <- kvEntry{ diff --git a/test/e2e/testdata/watermark.yaml b/test/e2e/testdata/watermark.yaml index a73a6a427c..777699045d 100644 --- a/test/e2e/testdata/watermark.yaml +++ b/test/e2e/testdata/watermark.yaml @@ -4,7 +4,7 @@ metadata: name: simple-pipeline-watermark spec: watermark: - propagate: true + disabled: false vertices: - name: input source: