From f8170577eada6972e64f5d338b9d387f23111e47 Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Thu, 17 Nov 2022 14:02:15 -0500 Subject: [PATCH] refactor(docs): group docs in categories (#362) Signed-off-by: Derek Wang --- docs/{ => development}/debugging.md | 0 docs/{ => development}/development.md | 0 docs/{ => development}/releasing.md | 0 .../{ => development}/static-code-analysis.md | 0 docs/{ => operations}/controller-configmap.md | 2 +- docs/{ => operations}/installation.md | 0 docs/{ => operations}/metrics/metrics.md | 0 .../numaflow-controller-config.yaml | 0 docs/{ => operations}/releases.md | 0 docs/quick-start.md | 16 +++-- docs/{specs => specifications}/autoscaling.md | 2 +- docs/{specs => specifications}/overview.md | 0 docs/{ => user-guide}/autoscaling.md | 0 .../builtin-functions/README.md | 0 .../{ => user-guide}/builtin-functions/cat.md | 0 .../builtin-functions/filter.md | 0 .../conditional-forwarding.md | 0 docs/{ => user-guide}/container-resources.md | 0 .../{ => user-guide}/environment-variables.md | 2 +- docs/{ => user-guide}/init-containers.md | 0 .../inter-step-buffer-service.md | 12 ++-- docs/{ => user-guide}/inter-step-buffer.md | 0 .../pipeline-customization.md | 0 docs/{ => user-guide}/pipeline-tuning.md | 0 docs/{ => user-guide}/pipeline.md | 0 docs/{ => user-guide}/sidecar-containers.md | 0 docs/{ => user-guide}/sinks/blackhole.md | 0 docs/{ => user-guide}/sinks/kafka.md | 0 docs/{ => user-guide}/sinks/log.md | 0 .../sinks/user-defined-sinks.md | 0 docs/{ => user-guide}/sources/generator.md | 0 docs/{ => user-guide}/sources/http.md | 0 docs/{ => user-guide}/sources/kafka.md | 0 .../user-defined-functions.md | 0 docs/{ => user-guide}/vertex.md | 0 docs/{ => user-guide}/volumes.md | 0 docs/{ => user-guide}/watermarks.md | 0 examples/0-isbsvc-jetstream.yaml | 2 +- mkdocs.yml | 70 +++++++++---------- 39 files changed, 57 insertions(+), 49 deletions(-) rename docs/{ => development}/debugging.md (100%) rename docs/{ => development}/development.md (100%) rename docs/{ => development}/releasing.md (100%) rename docs/{ => development}/static-code-analysis.md (100%) rename docs/{ => operations}/controller-configmap.md (82%) rename docs/{ => operations}/installation.md (100%) rename docs/{ => operations}/metrics/metrics.md (100%) rename docs/{ => operations}/numaflow-controller-config.yaml (100%) rename docs/{ => operations}/releases.md (100%) rename docs/{specs => specifications}/autoscaling.md (98%) rename docs/{specs => specifications}/overview.md (100%) rename docs/{ => user-guide}/autoscaling.md (100%) rename docs/{ => user-guide}/builtin-functions/README.md (100%) rename docs/{ => user-guide}/builtin-functions/cat.md (100%) rename docs/{ => user-guide}/builtin-functions/filter.md (100%) rename docs/{ => user-guide}/conditional-forwarding.md (100%) rename docs/{ => user-guide}/container-resources.md (100%) rename docs/{ => user-guide}/environment-variables.md (94%) rename docs/{ => user-guide}/init-containers.md (100%) rename docs/{ => user-guide}/inter-step-buffer-service.md (96%) rename docs/{ => user-guide}/inter-step-buffer.md (100%) rename docs/{ => user-guide}/pipeline-customization.md (100%) rename docs/{ => user-guide}/pipeline-tuning.md (100%) rename docs/{ => user-guide}/pipeline.md (100%) rename docs/{ => user-guide}/sidecar-containers.md (100%) rename docs/{ => user-guide}/sinks/blackhole.md (100%) rename docs/{ => user-guide}/sinks/kafka.md (100%) rename docs/{ => user-guide}/sinks/log.md (100%) rename docs/{ => user-guide}/sinks/user-defined-sinks.md (100%) rename docs/{ => user-guide}/sources/generator.md (100%) rename docs/{ => user-guide}/sources/http.md (100%) rename docs/{ => user-guide}/sources/kafka.md (100%) rename docs/{ => user-guide}/user-defined-functions.md (100%) rename docs/{ => user-guide}/vertex.md (100%) rename docs/{ => user-guide}/volumes.md (100%) rename docs/{ => user-guide}/watermarks.md (100%) diff --git a/docs/debugging.md b/docs/development/debugging.md similarity index 100% rename from docs/debugging.md rename to docs/development/debugging.md diff --git a/docs/development.md b/docs/development/development.md similarity index 100% rename from docs/development.md rename to docs/development/development.md diff --git a/docs/releasing.md b/docs/development/releasing.md similarity index 100% rename from docs/releasing.md rename to docs/development/releasing.md diff --git a/docs/static-code-analysis.md b/docs/development/static-code-analysis.md similarity index 100% rename from docs/static-code-analysis.md rename to docs/development/static-code-analysis.md diff --git a/docs/controller-configmap.md b/docs/operations/controller-configmap.md similarity index 82% rename from docs/controller-configmap.md rename to docs/operations/controller-configmap.md index 0f4161cef3..6a4f2c307e 100644 --- a/docs/controller-configmap.md +++ b/docs/operations/controller-configmap.md @@ -22,7 +22,7 @@ data: ### ISB Service Configuration -One of the important configuration items in the ConfigMap is about [ISB Service](./inter-step-buffer-service.md). We currently use 3rd party technologies such as `JetStream` to implement ISB Services, if those applications have new releases, to make them available in Numaflow, the new versions need to be added in the ConfigMap. +One of the important configuration items in the ConfigMap is about [ISB Service](../user-guide/inter-step-buffer-service.md). We currently use 3rd party technologies such as `JetStream` to implement ISB Services, if those applications have new releases, to make them available in Numaflow, the new versions need to be added in the ConfigMap. For example, there's a new `Nats JetStream` version `x.y.x` available, a new version configuration like below needs to be added before it can be referenced in the `InterStepBufferService` spec. diff --git a/docs/installation.md b/docs/operations/installation.md similarity index 100% rename from docs/installation.md rename to docs/operations/installation.md diff --git a/docs/metrics/metrics.md b/docs/operations/metrics/metrics.md similarity index 100% rename from docs/metrics/metrics.md rename to docs/operations/metrics/metrics.md diff --git a/docs/numaflow-controller-config.yaml b/docs/operations/numaflow-controller-config.yaml similarity index 100% rename from docs/numaflow-controller-config.yaml rename to docs/operations/numaflow-controller-config.yaml diff --git a/docs/releases.md b/docs/operations/releases.md similarity index 100% rename from docs/releases.md rename to docs/operations/releases.md diff --git a/docs/quick-start.md b/docs/quick-start.md index 22d208947c..796eeee3f9 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -16,6 +16,7 @@ You will also need `kubectl` to manage the cluster. ## Installation Run the following command lines to install Numaflow and start the Inter-Step Buffer Service that handles communication between vertices. + ```shell kubectl create ns numaflow-system kubectl apply -n numaflow-system -f https://raw.githubusercontent.com/numaproj/numaflow/stable/config/install.yaml @@ -25,6 +26,7 @@ kubectl apply -f https://raw.githubusercontent.com/numaproj/numaflow/stable/exam ## A Simple Pipeline Create a `simple pipeline`, which contains a source vertex to generate messages, a processing vertex that echos the messages, and a sink vertex that logs the messages. + ```shell kubectl apply -f https://raw.githubusercontent.com/numaproj/numaflow/stable/examples/1-simple-pipeline.yaml kubectl get pipeline # or "pl" as a short name @@ -56,20 +58,23 @@ kubectl logs -f simple-pipeline-out-0-xxxx # Port forward the UI to https://localhost:8443/ kubectl -n numaflow-system port-forward deployment/numaflow-server 8443:8443 ``` + ![Numaflow UI](assets/numaflow-ui-simple-pipeline.png) The pipeline can be deleted by + ```shell kubectl delete -f https://raw.githubusercontent.com/numaproj/numaflow/stable/examples/1-simple-pipeline.yaml ``` ## An Advanced Pipeline -In this example, there are five vertices in a pipeline. An [HTTP](./sources/http.md) source vertex which serves an HTTP endpoint to receive numbers as source data, a [UDF](./user-defined-functions.md) vertex to tag the ingested numbers with the key `even` or `odd`, three [Log](./sinks/log.md) sinks, one to print the `even` numbers, one to print the `odd` numbers, and the other one to print both the even and odd numbers. +In this example, there are five vertices in a pipeline. An [HTTP](./user-guide/sources/http.md) source vertex which serves an HTTP endpoint to receive numbers as source data, a [UDF](./user-guide/user-defined-functions.md) vertex to tag the ingested numbers with the key `even` or `odd`, three [Log](./user-guide/sinks/log.md) sinks, one to print the `even` numbers, one to print the `odd` numbers, and the other one to print both the even and odd numbers. ![Pipeline Diagram](assets/even-odd.png) Create the `even-odd` pipeline. + ```shell kubectl apply -f https://raw.githubusercontent.com/numaproj/numaflow/stable/test/e2e/testdata/even-odd.yaml @@ -91,9 +96,9 @@ kubectl port-forward even-odd-in-0-xxxx 8444:8443 # Post data to the HTTP endpoint curl -kq -X POST -d "101" https://localhost:8444/vertices/in -curl -kq -X POST -d "102" https://localhost:8444/vertices/in +curl -kq -X POST -d "102" https://localhost:8444/vertices/in curl -kq -X POST -d "103" https://localhost:8444/vertices/in -curl -kq -X POST -d "104" https://localhost:8444/vertices/in +curl -kq -X POST -d "104" https://localhost:8444/vertices/in # Watch the log for the even vertex kubectl logs -f even-odd-even-sink-0-nf2ql @@ -109,9 +114,10 @@ kubectl logs -f even-odd-odd-sink-0-a6p0n View the UI for the advanced pipeline at https://localhost:8443/ ![Numaflow UI](assets/numaflow-ui-advanced-pipeline.png) -The source code of the `even-odd` [User Defined Function](./user-defined-functions.md) can be found [here](https://github.com/numaproj/numaflow-go/tree/main/pkg/function/examples/evenodd). You also can replace the [Log](./sinks/log.md) Sink with some other sinks like [Kafka](./sinks/kafka.md) to forward the data to Kafka topics. +The source code of the `even-odd` [User Defined Function](./user-guide/user-defined-functions.md) can be found [here](https://github.com/numaproj/numaflow-go/tree/main/pkg/function/examples/evenodd). You also can replace the [Log](./user-guide/sinks/log.md) Sink with some other sinks like [Kafka](./user-guide/sinks/kafka.md) to forward the data to Kafka topics. The pipeline can be deleted by + ```shell kubectl delete -f https://raw.githubusercontent.com/numaproj/numaflow/stable/test/e2e/testdata/even-odd.yaml ``` @@ -120,4 +126,4 @@ kubectl delete -f https://raw.githubusercontent.com/numaproj/numaflow/stable/tes Try more examples in the [`examples`](https://github.com/numaproj/numaflow/tree/main/examples) directory. -After exploring how Numaflow pipeline run, you can check what data [Sources](./sources/generator.md) and [Sinks](./sinks/kafka.md) Numaflow supports out of the box, or learn how to write [User Defined Functions](./user-defined-functions.md). +After exploring how Numaflow pipeline run, you can check what data [Sources](./user-guide/sources/generator.md) and [Sinks](./user-guide/sinks/kafka.md) Numaflow supports out of the box, or learn how to write [User Defined Functions](./user-guide/user-defined-functions.md). diff --git a/docs/specs/autoscaling.md b/docs/specifications/autoscaling.md similarity index 98% rename from docs/specs/autoscaling.md rename to docs/specifications/autoscaling.md index 7f1df48fd6..dcf51474f7 100644 --- a/docs/specs/autoscaling.md +++ b/docs/specifications/autoscaling.md @@ -62,4 +62,4 @@ When the calculated desired replicas is greater than current replicas: ## Autoscaling Tuning -Numaflow autoscaling can be tuned by updating some parameters, find the details at the [doc](../autoscaling.md). +Numaflow autoscaling can be tuned by updating some parameters, find the details at the [doc](../user-guide/autoscaling.md). diff --git a/docs/specs/overview.md b/docs/specifications/overview.md similarity index 100% rename from docs/specs/overview.md rename to docs/specifications/overview.md diff --git a/docs/autoscaling.md b/docs/user-guide/autoscaling.md similarity index 100% rename from docs/autoscaling.md rename to docs/user-guide/autoscaling.md diff --git a/docs/builtin-functions/README.md b/docs/user-guide/builtin-functions/README.md similarity index 100% rename from docs/builtin-functions/README.md rename to docs/user-guide/builtin-functions/README.md diff --git a/docs/builtin-functions/cat.md b/docs/user-guide/builtin-functions/cat.md similarity index 100% rename from docs/builtin-functions/cat.md rename to docs/user-guide/builtin-functions/cat.md diff --git a/docs/builtin-functions/filter.md b/docs/user-guide/builtin-functions/filter.md similarity index 100% rename from docs/builtin-functions/filter.md rename to docs/user-guide/builtin-functions/filter.md diff --git a/docs/conditional-forwarding.md b/docs/user-guide/conditional-forwarding.md similarity index 100% rename from docs/conditional-forwarding.md rename to docs/user-guide/conditional-forwarding.md diff --git a/docs/container-resources.md b/docs/user-guide/container-resources.md similarity index 100% rename from docs/container-resources.md rename to docs/user-guide/container-resources.md diff --git a/docs/environment-variables.md b/docs/user-guide/environment-variables.md similarity index 94% rename from docs/environment-variables.md rename to docs/user-guide/environment-variables.md index e13b93cbf5..1afdea7c95 100644 --- a/docs/environment-variables.md +++ b/docs/user-guide/environment-variables.md @@ -1,6 +1,6 @@ # Environment Variables -For the `numa` container of vertex pods, environment variable `NUMAFLOW_DEBUG` can be set to `true` for [debugging](./debugging.md). +For the `numa` container of vertex pods, environment variable `NUMAFLOW_DEBUG` can be set to `true` for [debugging](../development/debugging.md). In [`udf`](./user-defined-functions.md) and [`udsink`](./sinks/user-defined-sinks.md) containers, there are some preset environment variables that can be used directly. diff --git a/docs/init-containers.md b/docs/user-guide/init-containers.md similarity index 100% rename from docs/init-containers.md rename to docs/user-guide/init-containers.md diff --git a/docs/inter-step-buffer-service.md b/docs/user-guide/inter-step-buffer-service.md similarity index 96% rename from docs/inter-step-buffer-service.md rename to docs/user-guide/inter-step-buffer-service.md index 7d04188911..90b6825a0f 100644 --- a/docs/inter-step-buffer-service.md +++ b/docs/user-guide/inter-step-buffer-service.md @@ -25,7 +25,9 @@ spec: # Optional, if not specified, defaults to "default" interStepBufferServiceName: different-name ``` + To query `Inter-Step Buffer Service` objects with `kubectl`: + ```sh kubectl get isbsvc ``` @@ -137,7 +139,7 @@ Once a JetStream ISB Service is created, toggling the `encryption` field will ca ### Other Configuration -Check [here](APIs.md#numaflow.numaproj.io/v1alpha1.JetStreamBufferService) for the full spec of `spec.jetstream`. +Check [here](../APIs.md#numaflow.numaproj.io/v1alpha1.JetStreamBufferService) for the full spec of `spec.jetstream`. ## Redis @@ -147,6 +149,7 @@ Check [here](APIs.md#numaflow.numaproj.io/v1alpha1.JetStreamBufferService) for t We also support external redis. #### External Redis + If you have a managed Redis, say in AWS, etc., we can make that Redis your ISB. All you need to do is provide the external Redis endpoint name. ```yaml @@ -163,15 +166,14 @@ spec: ### Cluster Mode -We support [cluster mode](https://redis.io/docs/reference/cluster-spec/), only if the Redis is an external managed Redis. -You will have to enter the `url` twice to indicate that the mode is cluster. This is because we use `Universal Client` which +We support [cluster mode](https://redis.io/docs/reference/cluster-spec/), only if the Redis is an external managed Redis. +You will have to enter the `url` twice to indicate that the mode is cluster. This is because we use `Universal Client` which requires more than one address to indicate the Redis is in cluster mode. ```yaml url: "numaflow-redis-cluster-0.numaflow-redis-cluster-headless:6379,numaflow-redis-cluster-1.numaflow-redis-cluster-headless:6379" ``` - ### Version Property `spec.redis.native.version` is required for a `native` Redis `InterStepBufferService`. Supported versions can be found from the ConfigMap `numaflow-controller-config` in the control plane namespace. @@ -243,4 +245,4 @@ Here is the [reference](https://github.com/redis/redis/blob/unstable/redis.conf) ### Other Configuration -Check [here](APIs.md#numaflow.numaproj.io/v1alpha1.NativeRedis) for the full spec of `spec.redis.native`. +Check [here](../APIs.md#numaflow.numaproj.io/v1alpha1.NativeRedis) for the full spec of `spec.redis.native`. diff --git a/docs/inter-step-buffer.md b/docs/user-guide/inter-step-buffer.md similarity index 100% rename from docs/inter-step-buffer.md rename to docs/user-guide/inter-step-buffer.md diff --git a/docs/pipeline-customization.md b/docs/user-guide/pipeline-customization.md similarity index 100% rename from docs/pipeline-customization.md rename to docs/user-guide/pipeline-customization.md diff --git a/docs/pipeline-tuning.md b/docs/user-guide/pipeline-tuning.md similarity index 100% rename from docs/pipeline-tuning.md rename to docs/user-guide/pipeline-tuning.md diff --git a/docs/pipeline.md b/docs/user-guide/pipeline.md similarity index 100% rename from docs/pipeline.md rename to docs/user-guide/pipeline.md diff --git a/docs/sidecar-containers.md b/docs/user-guide/sidecar-containers.md similarity index 100% rename from docs/sidecar-containers.md rename to docs/user-guide/sidecar-containers.md diff --git a/docs/sinks/blackhole.md b/docs/user-guide/sinks/blackhole.md similarity index 100% rename from docs/sinks/blackhole.md rename to docs/user-guide/sinks/blackhole.md diff --git a/docs/sinks/kafka.md b/docs/user-guide/sinks/kafka.md similarity index 100% rename from docs/sinks/kafka.md rename to docs/user-guide/sinks/kafka.md diff --git a/docs/sinks/log.md b/docs/user-guide/sinks/log.md similarity index 100% rename from docs/sinks/log.md rename to docs/user-guide/sinks/log.md diff --git a/docs/sinks/user-defined-sinks.md b/docs/user-guide/sinks/user-defined-sinks.md similarity index 100% rename from docs/sinks/user-defined-sinks.md rename to docs/user-guide/sinks/user-defined-sinks.md diff --git a/docs/sources/generator.md b/docs/user-guide/sources/generator.md similarity index 100% rename from docs/sources/generator.md rename to docs/user-guide/sources/generator.md diff --git a/docs/sources/http.md b/docs/user-guide/sources/http.md similarity index 100% rename from docs/sources/http.md rename to docs/user-guide/sources/http.md diff --git a/docs/sources/kafka.md b/docs/user-guide/sources/kafka.md similarity index 100% rename from docs/sources/kafka.md rename to docs/user-guide/sources/kafka.md diff --git a/docs/user-defined-functions.md b/docs/user-guide/user-defined-functions.md similarity index 100% rename from docs/user-defined-functions.md rename to docs/user-guide/user-defined-functions.md diff --git a/docs/vertex.md b/docs/user-guide/vertex.md similarity index 100% rename from docs/vertex.md rename to docs/user-guide/vertex.md diff --git a/docs/volumes.md b/docs/user-guide/volumes.md similarity index 100% rename from docs/volumes.md rename to docs/user-guide/volumes.md diff --git a/docs/watermarks.md b/docs/user-guide/watermarks.md similarity index 100% rename from docs/watermarks.md rename to docs/user-guide/watermarks.md diff --git a/examples/0-isbsvc-jetstream.yaml b/examples/0-isbsvc-jetstream.yaml index 609af9c14d..f770c27bb7 100644 --- a/examples/0-isbsvc-jetstream.yaml +++ b/examples/0-isbsvc-jetstream.yaml @@ -1,6 +1,6 @@ # # This manifest is intended for demonstration purpose, it's not suitable for production. -# Check https://numaflow.numaproj.io/inter-step-buffer-service/ to figure out reliable configuration for production. +# Check https://numaflow.numaproj.io/user-guide/inter-step-buffer-service/ to figure out reliable configuration for production. # apiVersion: numaflow.numaproj.io/v1alpha1 kind: InterStepBufferService diff --git a/mkdocs.yml b/mkdocs.yml index 137913429c..4b2ba39cd1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -40,48 +40,48 @@ nav: - quick-start.md - User Guide: - Core Concepts: - - pipeline.md - - vertex.md - - inter-step-buffer.md - - inter-step-buffer-service.md + - user-guide/pipeline.md + - user-guide/vertex.md + - user-guide/inter-step-buffer.md + - user-guide/inter-step-buffer-service.md - Sources: - - sources/generator.md - - sources/http.md - - sources/kafka.md + - user-guide/sources/generator.md + - user-guide/sources/http.md + - user-guide/sources/kafka.md - Sinks: - - sinks/kafka.md - - sinks/log.md - - sinks/blackhole.md - - User Defined Sinks: "sinks/user-defined-sinks.md" + - user-guide/sinks/kafka.md + - user-guide/sinks/log.md + - user-guide/sinks/blackhole.md + - User Defined Sinks: "user-guide/sinks/user-defined-sinks.md" - User Defined Functions: - - Overview: "user-defined-functions.md" + - Overview: "user-guide/user-defined-functions.md" - Built-in UDFs: - - Overview: "builtin-functions/README.md" - - Cat: "builtin-functions/cat.md" - - Filter: "builtin-functions/filter.md" - - pipeline-tuning.md - - conditional-forwarding.md - - autoscaling.md - - container-resources.md - - volumes.md - - environment-variables.md - - init-containers.md - - sidecar-containers.md - - pipeline-customization.md - - watermarks.md + - Overview: "user-guide/builtin-functions/README.md" + - Cat: "user-guide/builtin-functions/cat.md" + - Filter: "user-guide/builtin-functions/filter.md" + - user-guide/pipeline-tuning.md + - user-guide/conditional-forwarding.md + - user-guide/autoscaling.md + - user-guide/container-resources.md + - user-guide/volumes.md + - user-guide/environment-variables.md + - user-guide/init-containers.md + - user-guide/sidecar-containers.md + - user-guide/pipeline-customization.md + - user-guide/watermarks.md - APIs.md - Operator Manual: - - Releases ⧉: "releases.md" - - installation.md + - Releases ⧉: "operations/releases.md" + - operations/installation.md - Configuration: - - Controller Configuration: "controller-configmap.md" - - metrics/metrics.md + - Controller Configuration: "operations/controller-configmap.md" + - operations/metrics/metrics.md - Developer Guide: - - development.md + - development/development.md - Specifications: - - Overview: "specs/overview.md" - - "specs/autoscaling.md" - - debugging.md - - static-code-analysis.md - - releasing.md + - Overview: "specifications/overview.md" + - "specifications/autoscaling.md" + - development/debugging.md + - development/static-code-analysis.md + - development/releasing.md - Numaproj: https://numaproj.io