Skip to content

Commit f82ddaa

Browse files
gaopenghighgaopenghigh
authored
Merge pull request #9 from ucloud/v0.1.7
V0.1.7
2 parents 78b14d0 + 4f4ef2e commit f82ddaa

File tree

1,505 files changed

+165093
-178228
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,505 files changed

+165093
-178228
lines changed

‎Dockerfile

+10-7
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1-
FROM golang:1.11.5-alpine3.7 as go-builder
1+
FROM golang:1.13.3-alpine as go-builder
2+
3+
RUN apk update && apk upgrade && \
4+
apk add --no-cache ca-certificates git mercurial
25

36
ARG PROJECT_NAME=redis-operator
47
ARG REPO_PATH=github.com/ucloud/$PROJECT_NAME
@@ -8,21 +11,21 @@ ARG BUILD_PATH=${REPO_PATH}/cmd/manager
811
ARG VERSION=0.1.1
912
ARG GIT_SHA=0000000
1013

11-
RUN mkdir -p /go/src/${REPO_PATH}/vendor
14+
WORKDIR /src
15+
16+
COPY go.mod ./ go.sum ./
17+
RUN GOPROXY=https://goproxy.cn,direct go mod download
1218

13-
COPY pkg /go/src/${REPO_PATH}/pkg
14-
COPY cmd /go/src/${REPO_PATH}/cmd
15-
COPY vendor /go/src/${REPO_PATH}/vendor
19+
COPY pkg ./ cmd ./ version ./
1620

1721
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ${GOBIN}/${PROJECT_NAME} \
1822
-ldflags "-X ${REPO_PATH}/pkg/version.Version=${VERSION} -X ${REPO_PATH}/pkg/version.GitSHA=${GIT_SHA}" \
1923
$BUILD_PATH
2024

2125
# =============================================================================
22-
FROM alpine:3.7 AS final
26+
FROM alpine:3.9 AS final
2327

2428
ARG PROJECT_NAME=redis-operator
25-
ARG REPO_PATH=github.com/ucloud/$PROJECT_NAME
2629

2730
COPY --from=go-builder ${GOBIN}/${PROJECT_NAME} /usr/local/bin/${PROJECT_NAME}
2831

‎Dockerfile-withvendor

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
FROM golang:1.13.3-alpine as go-builder
2+
3+
ARG PROJECT_NAME=redis-operator
4+
ARG REPO_PATH=github.com/ucloud/$PROJECT_NAME
5+
ARG BUILD_PATH=${REPO_PATH}/cmd/manager
6+
7+
# Build version and commit should be passed in when performing docker build
8+
ARG VERSION=0.1.1
9+
ARG GIT_SHA=0000000
10+
11+
RUN mkdir -p /go/src/${REPO_PATH}/vendor
12+
13+
COPY pkg /go/src/${REPO_PATH}/pkg
14+
COPY cmd /go/src/${REPO_PATH}/cmd
15+
COPY vendor /go/src/${REPO_PATH}/vendor
16+
17+
RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ${GOBIN}/${PROJECT_NAME} \
18+
-ldflags "-X ${REPO_PATH}/pkg/version.Version=${VERSION} -X ${REPO_PATH}/pkg/version.GitSHA=${GIT_SHA}" \
19+
$BUILD_PATH
20+
21+
# =============================================================================
22+
FROM alpine:3.9 AS final
23+
24+
ARG PROJECT_NAME=redis-operator
25+
ARG REPO_PATH=github.com/ucloud/$PROJECT_NAME
26+
27+
COPY --from=go-builder ${GOBIN}/${PROJECT_NAME} /usr/local/bin/${PROJECT_NAME}
28+
29+
RUN adduser -D ${PROJECT_NAME}
30+
USER ${PROJECT_NAME}
31+
32+
ENTRYPOINT ["/usr/local/bin/redis-operator"]

‎Makefile

+3-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ E2EALTREPO=$(REGISTRY)/$(PROJECT_NAME)-e2e
1010
VERSION=$(shell git describe --always --tags --dirty | sed "s/\(.*\)-g`git rev-parse --short HEAD`/\1/")
1111
GIT_SHA=$(shell git rev-parse --short HEAD)
1212
BIN_DIR=build/bin
13-
.PHONY: all build check clean test login
13+
.PHONY: all build check clean test login build-e2e push-e2e
1414

1515
all: check build
1616

@@ -28,6 +28,7 @@ build-image:
2828
docker build --build-arg VERSION=$(VERSION) --build-arg GIT_SHA=$(GIT_SHA) -t $(ALTREPO):$(VERSION) .
2929
docker tag $(ALTREPO):$(VERSION) $(ALTREPO):latest
3030

31+
build-e2e:
3132
docker build -t $(E2EALTREPO):$(VERSION) -f test/e2e/Dockerfile .
3233

3334
test:
@@ -40,6 +41,7 @@ push: build-image
4041
docker push $(ALTREPO):$(VERSION)
4142
docker push $(ALTREPO):latest
4243

44+
push-e2e: build-e2e
4345
docker push $(E2EALTREPO):$(VERSION)
4446

4547
clean:

‎README.md

+92-30
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,37 @@ The operator itself is built with the [Operator framework](https://github.com/op
99

1010
It inspired by [spotahome/redis-operator](https://github.com/spotahome/redis-operator).
1111

12+
![Redis Cluster atop Kubernetes](/static/redis-sentinel-readme.png)
13+
14+
* Create a statefulset to mange Redis instances (masters and replicas), each redis instance has default PreStop script that can do failover if master is down.
15+
* Create a statefulset to mange Sentinel instances that will control the Redis nodes, each Sentinel instance has default ReadinessProbe script to detect whether the current sentinel's status is ok. When a sentinel pod is not ready, it is removed from Service load balancers.
16+
* Create a Service and a Headless service for Sentinel statefulset.
17+
* Create a Headless service for Redis statefulset.
18+
19+
Table of Contents
20+
=================
21+
22+
* [redis-operator](#redis-operator)
23+
* [Overview](#overview)
24+
* [Prerequisites](#prerequisites)
25+
* [Features](#features)
26+
* [Quick Start](#quick-start)
27+
* [Deploy redis operator](#deploy-redis-operator)
28+
* [Deploy a sample redis cluster](#deploy-a-sample-redis-cluster)
29+
* [Resize an Redis Cluster](#resize-an-redis-cluster)
30+
* [Create redis cluster with password](#create-redis-cluster-with-password)
31+
* [Dynamically changing redis config](#dynamically-changing-redis-config)
32+
* [Persistence](#persistence)
33+
* [Custom SecurityContext](#custom-securitycontext)
34+
* [Cleanup](#cleanup)
35+
* [Automatic failover details](#automatic-failover-details)
36+
1237
## Prerequisites
1338

14-
* go version v1.12+.
15-
* Access to a Kubernetes v1.11.3+ cluster.
39+
* go version v1.13+.
40+
* Access to a Kubernetes v1.13.10+ cluster.
1641

17-
## Capabilities
42+
## Features
1843
In addition to the sentinel's own capabilities, redis-operator can:
1944

2045
* Push events and update status to the Kubernetes when resources have state changes
@@ -28,7 +53,7 @@ In addition to the sentinel's own capabilities, redis-operator can:
2853
## Quick Start
2954

3055
### Deploy redis operator
31-
Build and push the redis-operator and e2e test image
56+
Build and push the redis-operator image
3257
```
3358
$ make REGISTRY=you_public_registry build-image
3459
$ make REGISTRY=you_public_registry push
@@ -92,33 +117,32 @@ Verify that the cluster instances and its components are running.
92117
```
93118
$ kubectl get rediscluster
94119
NAME SIZE STATUS AGE
95-
test 3 Healthy 22h
120+
test 3 Healthy 4m9s
96121
97122
$ kubectl get all -l app.kubernetes.io/managed-by=redis-operator
98-
NAME READY STATUS RESTARTS AGE
99-
pod/redis-cluster-test-0 1/1 Running 0 22h
100-
pod/redis-cluster-test-1 1/1 Running 0 22h
101-
pod/redis-cluster-test-2 1/1 Running 0 22h
102-
pod/redis-sentinel-test-7cbd85785b-6llfp 1/1 Running 0 22h
103-
pod/redis-sentinel-test-7cbd85785b-ggqw4 1/1 Running 0 22h
104-
pod/redis-sentinel-test-7cbd85785b-nxxfc 1/1 Running 0 22h
105-
106-
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
107-
service/redis-sentinel-test ClusterIP xxxxxxxxxx <none> 26379/TCP 22h
108-
109-
NAME READY UP-TO-DATE AVAILABLE AGE
110-
deployment.apps/redis-sentinel-test 3/3 3 3 22h
111-
112-
NAME DESIRED CURRENT READY AGE
113-
replicaset.apps/redis-sentinel-test-7cbd85785b 3 3 3 22h
114-
115-
NAME READY AGE
116-
statefulset.apps/redis-cluster-test 3/3 22h
123+
NAME READY STATUS RESTARTS AGE
124+
pod/redis-cluster-test-0 1/1 Running 0 4m16s
125+
pod/redis-cluster-test-1 1/1 Running 0 3m22s
126+
pod/redis-cluster-test-2 1/1 Running 0 2m40s
127+
pod/redis-sentinel-test-0 1/1 Running 0 4m16s
128+
pod/redis-sentinel-test-1 1/1 Running 0 81s
129+
pod/redis-sentinel-test-2 1/1 Running 0 18s
130+
131+
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
132+
service/redis-cluster-test ClusterIP None <none> 6379/TCP 4m16s
133+
service/redis-sentinel-headless-test ClusterIP None <none> 26379/TCP 4m16s
134+
service/redis-sentinel-test ClusterIP 10.22.22.34 <none> 26379/TCP 4m16s
135+
136+
NAME READY AGE
137+
statefulset.apps/redis-cluster-test 3/3 4m16s
138+
statefulset.apps/redis-sentinel-test 3/3 4m16s
117139
```
118140

119141
* redis-cluster-<NAME>: Redis statefulset
120-
* redis-sentinel-<NAME>: Sentinel deployment
142+
* redis-sentinel-<NAME>: Sentinel statefulset
121143
* redis-sentinel-<NAME>: Sentinel service
144+
* redis-sentinel-headless-<NAME>: Sentinel headless service
145+
* redis-cluster-<NAME>: Redis headless service
122146

123147
Describe the Redis Cluster, Viewing Events and Status
124148
```
@@ -277,18 +301,23 @@ spec:
277301
cpu: 50m
278302
memory: 30Mi
279303
size: 3
304+
280305
# when the disablePersistence set to false, the following configurations will be set automatically:
306+
307+
# disablePersistence: false
308+
# config["save"] = "900 1 300 10"
281309
# config["appendonly"] = "yes"
282-
# config["auto-aof-rewrite-min-size"] = "1gb"
310+
# config["auto-aof-rewrite-min-size"] = "536870912"
283311
# config["repl-diskless-sync"] = "yes"
284-
# config["repl-backlog-size"] = "60mb"
285-
# config["repl-diskless-sync-delay"] = "5"
312+
# config["repl-backlog-size"] = "62914560"
286313
# config["aof-load-truncated"] = "yes"
287314
# config["stop-writes-on-bgsave-error"] = "no"
315+
288316
# when the disablePersistence set to true, the following configurations will be set automatically:
317+
318+
# disablePersistence: true
289319
# config["save"] = ""
290320
# config["appendonly"] = "no"
291-
disablePersistence: false
292321
storage:
293322
# By default, the persistent volume claims will be deleted when the Redis Cluster be delete.
294323
# If this is not the expected usage, a keepAfterDeletion flag can be added under the storage section
@@ -344,4 +373,37 @@ $ kubectl delete -f deploy/namespace/role.yaml
344373
$ kubectl delete -f deploy/namespace/role_binding.yaml
345374
$ kubectl delete -f deploy/service_account.yaml
346375
$ kubectl delete -f deploy/crds/redis_v1beta1_rediscluster_crd.yaml
347-
```
376+
```
377+
378+
## Automatic failover details
379+
380+
Redis-operator build a **Highly Available Redis cluster with Sentinel**, Sentinel always checks the MASTER and SLAVE
381+
instances in the Redis cluster, checking whether they working as expected. If sentinel detects a failure in the
382+
MASTER node in a given cluster, Sentinel will start a failover process. As a result, Sentinel will pick a SLAVE
383+
instance and promote it to MASTER. Ultimately, the other remaining SLAVE instances will be automatically reconfigured
384+
to use the new MASTER instance.
385+
386+
operator guarantees the following:
387+
* Only one Redis instance as master in a cluster
388+
* Number of Redis instance(masters and replicas) is equal as the set on the RedisCluster specification
389+
* Number of Sentinels is equal as the set on the RedisCluster specification
390+
* All Redis slaves have the same master
391+
* All Sentinels point to the same Redis master
392+
* Sentinel has not dead nodes
393+
394+
But Kubernetes pods are volatile, they can be deleted and recreated, and pods IP will change when pod be recreated,
395+
and also, the IP will be recycled and redistributed to other pods.
396+
Unfortunately, sentinel cannot delete the sentinel list or redis list in its memory when the pods IP changes.
397+
This can be caused because there’s no way of a Sentinel node to self-deregister from the Sentinel Cluster before die,
398+
provoking the Sentinel node list to increase without any control.
399+
400+
To ensure that Sentinel is working properly, operator will send a **RESET(SENTINEL RESET * )** signal to Sentinel node
401+
one by one (if no failover is being running at that moment).
402+
`SENTINEL RESET mastername` command: they'll refresh the list of replicas within the next 10 seconds, only adding the
403+
ones listed as correctly replicating from the current master INFO output.
404+
During this refresh time, `SENTINEL slaves <master name>` command can not get any result from sentinel, so operator sent
405+
RESET signal to Sentinel one by one and wait sentinel status became ok(monitor correct master and has slaves).
406+
Additional, Each Sentinel instance has default ReadinessProbe script to detect whether the current sentinel's status is ok.
407+
When a sentinel pod is not ready, it is removed from Service load balancers.
408+
Operator also create a headless svc for Sentinel statefulset, if you can not get result from `SENTINEL slaves <master name>` command,
409+
You can try polling the headless domain.

‎go.mod

+32-24
Original file line numberDiff line numberDiff line change
@@ -3,33 +3,41 @@ module github.com/ucloud/redis-operator
33
require (
44
github.com/go-logr/logr v0.1.0
55
github.com/go-redis/redis v6.15.2+incompatible
6-
github.com/onsi/ginkgo v1.7.0
7-
github.com/onsi/gomega v1.4.3
8-
github.com/operator-framework/operator-sdk v0.8.1-0.20190711220207-283ac6f58bec
9-
github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
10-
github.com/spf13/pflag v1.0.3
11-
github.com/stretchr/testify v1.3.0
12-
k8s.io/api v0.0.0-20190612125737-db0771252981
13-
k8s.io/apimachinery v0.0.0-20190612125636-6a5db36e93ad
14-
k8s.io/client-go v11.0.0+incompatible
15-
k8s.io/kube-openapi v0.0.0-20190603182131-db7b694dc208 // indirect
16-
sigs.k8s.io/controller-runtime v0.1.12
17-
sigs.k8s.io/controller-tools v0.1.10
6+
github.com/onsi/ginkgo v1.8.0
7+
github.com/onsi/gomega v1.5.0
8+
github.com/operator-framework/operator-sdk v0.13.0
9+
github.com/prometheus/client_golang v1.1.0
10+
github.com/spf13/pflag v1.0.5
11+
github.com/stretchr/testify v1.4.0
12+
k8s.io/api v0.0.0
13+
k8s.io/apimachinery v0.0.0
14+
k8s.io/client-go v12.0.0+incompatible
15+
sigs.k8s.io/controller-runtime v0.4.0
1816
)
1917

20-
// Pinned to kubernetes-1.13.4
18+
// Pinned to kubernetes-1.16.2
2119
replace (
22-
k8s.io/api => k8s.io/api v0.0.0-20190222213804-5cb15d344471
23-
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20190228180357-d002e88f6236
24-
k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20190221213512-86fb29eff628
25-
k8s.io/client-go => k8s.io/client-go v0.0.0-20190228174230-b40b2a5939e4
26-
)
27-
28-
replace (
29-
github.com/coreos/prometheus-operator => github.com/coreos/prometheus-operator v0.29.0
30-
k8s.io/kube-state-metrics => k8s.io/kube-state-metrics v1.6.0
31-
sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.1.12
32-
sigs.k8s.io/controller-tools => sigs.k8s.io/controller-tools v0.1.11-0.20190411181648-9d55346c2bde
20+
k8s.io/api => k8s.io/api v0.0.0-20191016110408-35e52d86657a
21+
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20191016113550-5357c4baaf65
22+
k8s.io/apimachinery => k8s.io/apimachinery v0.0.0-20191004115801-a2eda9f80ab8
23+
k8s.io/apiserver => k8s.io/apiserver v0.0.0-20191016112112-5190913f932d
24+
k8s.io/cli-runtime => k8s.io/cli-runtime v0.0.0-20191016114015-74ad18325ed5
25+
k8s.io/client-go => k8s.io/client-go v0.0.0-20191016111102-bec269661e48
26+
k8s.io/cloud-provider => k8s.io/cloud-provider v0.0.0-20191016115326-20453efc2458
27+
k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.0.0-20191016115129-c07a134afb42
28+
k8s.io/code-generator => k8s.io/code-generator v0.0.0-20191004115455-8e001e5d1894
29+
k8s.io/component-base => k8s.io/component-base v0.0.0-20191016111319-039242c015a9
30+
k8s.io/cri-api => k8s.io/cri-api v0.0.0-20190828162817-608eb1dad4ac
31+
k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.0.0-20191016115521-756ffa5af0bd
32+
k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.0.0-20191016112429-9587704a8ad4
33+
k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.0.0-20191016114939-2b2b218dc1df
34+
k8s.io/kube-proxy => k8s.io/kube-proxy v0.0.0-20191016114407-2e83b6f20229
35+
k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.0.0-20191016114748-65049c67a58b
36+
k8s.io/kubectl => k8s.io/kubectl v0.0.0-20191016120415-2ed914427d51
37+
k8s.io/kubelet => k8s.io/kubelet v0.0.0-20191016114556-7841ed97f1b2
38+
k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.0.0-20191016115753-cf0698c3a16b
39+
k8s.io/metrics => k8s.io/metrics v0.0.0-20191016113814-3b1a734dba6e
40+
k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.0.0-20191016112829-06bb3c9d77c9
3341
)
3442

3543
go 1.13

0 commit comments

Comments
 (0)