chore: add mkdocs check (numaproj#166)

* chore: mkdocs check

Signed-off-by: Derek Wang <[email protected]>

.github/workflows/ci.yaml

@@ -42,7 +42,7 @@ jobs:
       - name: Setup Golang
         uses: actions/[email protected]
         with:
-          go-version: "1.18"
+          go-version: "1.19"
       - name: Add bins to PATH
         run: |
           echo /home/runner/go/bin >> $GITHUB_PATH
@@ -86,7 +86,7 @@ jobs:
       - name: Set up Go 1.x
         uses: actions/[email protected]
         with:
-          go-version: "1.18"
+          go-version: "1.19"
         id: go
       - name: Check out code into the Go module directory
         uses: actions/checkout@v3
@@ -123,7 +123,7 @@ jobs:
       - name: Setup Golang
         uses: actions/[email protected]
         with:
-          go-version: "1.18"
+          go-version: "1.19"
       - name: Restore Go build cache
         uses: actions/cache@v3
         with:
@@ -153,7 +153,7 @@ jobs:
       - name: Setup Golang
         uses: actions/[email protected]
         with:
-          go-version: "1.18"
+          go-version: "1.19"
       - name: Add bins to PATH
         run: |
           echo /home/runner/go/bin >> $GITHUB_PATH

.github/workflows/gh-pages.yaml

@@ -1,9 +1,12 @@
-name: gh-pages
+name: docs
 
 on:
   push:
     branches:
       - main
+  pull_request:
+    branches:
+      - main
 
 jobs:
   docs:
@@ -18,13 +21,12 @@ jobs:
       - name: Setup Golang
         uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: '1.19'
       - name: build
-        run: |
-          pip install mkdocs==1.3.0 mkdocs_material==8.3.9
-          mkdocs build
+        run: make docs
       - name: deploy
         uses: peaceiris/actions-gh-pages@v3
+        if: github.repository == 'numaproj/numaflow' && github.ref == 'refs/heads/main'
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./site

.github/workflows/release.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Setup Go
         uses: actions/[email protected]
         with:
-          go-version: 1.18
+          go-version: 1.19
 
       - name: Build binaries
         run: |
@@ -101,7 +101,7 @@ jobs:
           fi
       - uses: actions/[email protected]
         with:
-          go-version: 1.18
+          go-version: 1.19
       - uses: actions/checkout@v3
       - run: go install sigs.k8s.io/bom/cmd/[email protected]
       - run: go install github.com/spdx/spdx-sbom-generator/cmd/[email protected]

Makefile

@@ -32,6 +32,15 @@ VERSION=$(GIT_TAG)
 override LDFLAGS += -X ${PACKAGE}.gitTag=${GIT_TAG}
 endif
 
+# Check Python
+PYTHON:=$(shell command -v python 2> /dev/null)
+ifndef PYTHON
+PYTHON:=$(shell command -v python3 2> /dev/null)
+endif
+ifndef PYTHON
+$(error "Python is not available, please install.")
+endif
+
 K3D ?= $(shell [ "`command -v kubectl`" != '' ] && [ "`command -v k3d`" != '' ] && [[ "`kubectl config current-context`" =~ k3d-* ]] && echo true || echo false)
 
 .PHONY: build
@@ -176,7 +185,7 @@ manifests: crds
 	kubectl kustomize config/advanced-install/minimal-crds > config/advanced-install/minimal-crds.yaml
 
 $(GOPATH)/bin/golangci-lint:
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b `go env GOPATH`/bin v1.46.2
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b `go env GOPATH`/bin v1.49.0
 
 .PHONY: lint
 lint: $(GOPATH)/bin/golangci-lint
@@ -197,6 +206,19 @@ ifeq ($(K3D),true)
 	k3d image import $(IMAGE_NAMESPACE)/e2eapi:$(VERSION)
 endif
 
+/usr/local/bin/mkdocs:
+	$(PYTHON) -m pip install mkdocs==1.3.0 mkdocs_material==8.3.9
+
+# docs
+
+.PHONY: docs
+docs: /usr/local/bin/mkdocs
+	mkdocs build
+
+.PHONY: docs-serve
+docs-serve: docs
+	mkdocs serve
+
 # pre-push checks
 
 .git/hooks/%: hack/git/hooks/%

cmd/commands/commands_test.go

@@ -4,7 +4,7 @@ import (
 	"bytes"
 	"encoding/base64"
 	"encoding/json"
-	"io/ioutil"
+	"io"
 	"os"
 	"testing"
 
@@ -23,7 +23,7 @@ func Test_Commands(t *testing.T) {
 		rootCmd.SetOut(b)
 		rootCmd.SetArgs([]string{"help"})
 		Execute()
-		output, _ := ioutil.ReadAll(b)
+		output, _ := io.ReadAll(b)
 		assert.Contains(t, string(output), "Available Commands")
 	})
 

controllers/vertex/scaling/doc.go

@@ -22,5 +22,4 @@ limitations under the License.
 //
 // Function StartWatching() and StopWatching() are also provided in the package,
 // so that vertices can be added into and removed from the workqueue.
-//
 package scaling

controllers/vertex/scaling/scaling.go

@@ -108,11 +108,13 @@ func (s *Scaler) scale(ctx context.Context, id int, keyCh <-chan string) {
 // scaleOneVertex implements the detailed logic of scaling up/down a vertex.
 //
 // For source vertices which have both rate and pending message information,
-//    desiredReplicas = currentReplicas * pending / (targetProcessingTime * rate)
+//
+//	desiredReplicas = currentReplicas * pending / (targetProcessingTime * rate)
 //
 // For UDF and sinks which have the read buffer information
-//    singleReplicaContribution = (totalAvailableBufferLength - pending) / currentReplicas
-//    desiredReplicas = targetAvailableBufferLength / singleReplicaContribution
+//
+//	singleReplicaContribution = (totalAvailableBufferLength - pending) / currentReplicas
+//	desiredReplicas = targetAvailableBufferLength / singleReplicaContribution
 //
 // Back pressure factor
 // When desiredReplicas > currentReplics:

docs/development.md

@@ -4,7 +4,7 @@ This doc explains how to set up a development environment for Numaflow.
 
 ### Install required tools
 
-1. [`go`](https://golang.org/doc/install) 1.18+
+1. [`go`](https://golang.org/doc/install) 1.19+
 1. [`git`](https://help.github.com/articles/set-up-git/)
 1. [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/)
 1. [`protoc`](https://github.com/protocolbuffers/protobuf) for compiling protocol buffers
@@ -38,3 +38,9 @@ k3d kubeconfig get k3s-default
 
 - `make start`
   Build the source code, image, and install the Numa controller in the `numaflow-system` namespace.
+
+- `make docs`
+  Convert the docs to Github pages, check if there's any error.
+
+- `make docs-serve`
+  Start [an HTTP server](http://127.0.0.1:8000/) on your local to host the docs generated Github pages.

pkg/daemon/server/service/pipeline_metrics_query_test.go

@@ -3,7 +3,7 @@ package service
 import (
 	"bytes"
 	"context"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"testing"
 
@@ -73,7 +73,7 @@ vertex_pending_messages{period="1m",pipeline="simple-pipeline",vertex="cat"} 5.3
 vertex_pending_messages{period="5m",pipeline="simple-pipeline",vertex="cat"} 6.002
 vertex_pending_messages{period="default",pipeline="simple-pipeline",vertex="cat"} 7.00002
 `
-	ioReader := ioutil.NopCloser(bytes.NewReader([]byte(metricsResponse)))
+	ioReader := io.NopCloser(bytes.NewReader([]byte(metricsResponse)))
 
 	pipelineMetricsQueryService.httpClient = &mockHttpClient{
 		MockGet: func(url string) (*http.Response, error) {

pkg/isb/redis/read_test.go

@@ -381,7 +381,7 @@ func (suite *ReadWritePerformance) TestReadWriteLatency() {
 	<-stopped
 }
 
-//TestReadWriteLatencyPipelining is performs wthe latency test during a forward.
+// TestReadWriteLatencyPipelining is performs wthe latency test during a forward.
 func (suite *ReadWritePerformance) TestReadWriteLatencyPipelining() {
 	suite.rqw, _ = NewBufferWrite(suite.ctx, suite.rclient, "ReadWritePerformance-to", "ReadWritePerformance-group-to", WithInfoRefreshInterval(2*time.Second), WithLagDuration(time.Minute), WithoutPipelining(), WithMaxLength(20000)).(*BufferWrite)
 	_ = NewBufferRead(suite.ctx, suite.rclient, "ReadWritePerformance-to", "ReadWritePerformance-group-to", "consumer-0")

pkg/pbq/pbq.go

@@ -48,7 +48,7 @@ func (p *PBQ) Write(ctx context.Context, message *isb.Message) error {
 	return writeErr
 }
 
-//CloseOfBook closes output channel
+// CloseOfBook closes output channel
 func (p *PBQ) CloseOfBook() {
 	close(p.output)
 	p.cob = true

pkg/shared/clients/jetstream/doc.go

@@ -31,5 +31,4 @@ limitations under the License.
 // For example:
 //
 // client.Connect(ctx, AutoReconnect())
-//
 package jetstream

pkg/shared/clients/redis/redis_client.go

@@ -72,7 +72,7 @@ func (cl *RedisClient) StreamInfo(ctx context.Context, streamKey string) (*redis
 	return cl.Client.XInfoStream(ctx, streamKey).Result()
 }
 
-//StreamGroupInfo returns redis stream group info
+// StreamGroupInfo returns redis stream group info
 func (cl *RedisClient) StreamGroupInfo(ctx context.Context, streamKey string) ([]redis.XInfoGroup, error) {
 	return cl.Client.XInfoGroups(ctx, streamKey).Result()
 }

pkg/shared/util/tls_config.go

@@ -4,7 +4,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"fmt"
-	"io/ioutil"
+	"os"
 
 	dfv1 "github.com/numaproj/numaflow/pkg/apis/numaflow/v1alpha1"
 )
@@ -47,7 +47,7 @@ func GetTLSConfig(config *dfv1.TLS) (*tls.Config, error) {
 		InsecureSkipVerify: config.InsecureSkipVerify,
 	}
 	if len(caCertPath) > 0 {
-		caCert, err := ioutil.ReadFile(caCertPath)
+		caCert, err := os.ReadFile(caCertPath)
 		if err != nil {
 			return nil, fmt.Errorf("failed to read ca cert file %s, %w", caCertPath, err)
 		}

pkg/shared/util/volume.go

@@ -2,7 +2,7 @@ package util
 
 import (
 	"fmt"
-	"io/ioutil"
+	"os"
 	"reflect"
 	"strings"
 
@@ -21,7 +21,7 @@ func GetSecretFromVolume(selector *corev1.SecretKeySelector) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	data, err := ioutil.ReadFile(filePath)
+	data, err := os.ReadFile(filePath)
 	if err != nil {
 		return "", fmt.Errorf("failed to get secret value of name: %s, key: %s, %w", selector.Name, selector.Key, err)
 	}
@@ -45,7 +45,7 @@ func GetConfigMapFromVolume(selector *corev1.ConfigMapKeySelector) (string, erro
 	if err != nil {
 		return "", err
 	}
-	data, err := ioutil.ReadFile(filePath)
+	data, err := os.ReadFile(filePath)
 	if err != nil {
 		return "", fmt.Errorf("failed to get configMap value of name: %s, key: %s, %w", selector.Name, selector.Key, err)
 	}

pkg/sinks/udsink/udsink_http.go

@@ -6,7 +6,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"time"
@@ -124,10 +123,10 @@ retry:
 			}
 			return errs
 		} else {
-			data, err = ioutil.ReadAll(resp.Body)
+			data, err = io.ReadAll(resp.Body)
 			if err != nil {
 				failed = true
-				logging.FromContext(ctx).Warnf("ioutil.ReadAll failed (%d/%d), %w", i, retryCount, err)
+				logging.FromContext(ctx).Warnf("io.ReadAll failed (%d/%d), %w", i, retryCount, err)
 				goto retry
 			}
 			_ = resp.Body.Close()

pkg/sources/generator/tickgen.go

@@ -114,7 +114,9 @@ func WithReadTimeOut(timeout time.Duration) Option {
 
 // NewMemGen fuction creates an instance of generator.
 // ctx  - context passed by the cmd/start.go a new context with cancel
-//        is created for use by this vertex.
+//
+//	is created for use by this vertex.
+//
 // name - name of this vertex
 // rpu  - no of records to generate per time unit. by default the channel buffer size is set to 5*rpu
 // msgSize - size of each generated message
@@ -197,7 +199,6 @@ func (mg *memgen) IsEmpty() bool {
 // There is a chance that we have read the message and the container got forcefully terminated before processing. To provide
 // at-least-once semantics for reading, during restart we will have to reprocess all unacknowledged messages.
 //
-//
 // the context passed to read should be different from the lifecycle context that is used by this vertex.
 func (mg *memgen) Read(ctx context.Context, count int64) ([]*isb.ReadMessage, error) {
 	msgs := make([]*isb.ReadMessage, 0, count)

pkg/sources/http/http.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"net/http"
 	"time"
 
@@ -106,7 +106,7 @@ func New(vertex *dfv1.Vertex, writers []isb.BufferWriter, opts ...Option) (*http
 			_, _ = w.Write([]byte("503 not ready\n"))
 			return
 		}
-		msg, err := ioutil.ReadAll(r.Body)
+		msg, err := io.ReadAll(r.Body)
 		_ = r.Body.Close()
 		if err != nil {
 			w.WriteHeader(400)