*: rebuild from special pod #652 (#658)

Author: acekingke

config/samples/nfs_server.yaml

@@ -14,7 +14,7 @@ spec:
       containers:
       - name: nfs-server
         ## TODO: radondb/volume-nfs:0.8
-        image: gcr.azk8s.cn/google_containers/volume-nfs:0.8
+        image: k8s.gcr.io/volume-nfs:0.8
         ports:
           - name: nfs
             containerPort: 2049

mysqlcluster/syncer/role.go

@@ -42,7 +42,7 @@ func NewRoleSyncer(cli client.Client, c *mysqlcluster.MysqlCluster) syncer.Inter
 	return syncer.NewObjectSyncer("Role", c.Unwrap(), role, cli, func() error {
 		role.Rules = []rbacv1.PolicyRule{
 			{
-				Verbs:     []string{"get", "patch"},
+				Verbs:     []string{"get", "patch", "list"},
 				APIGroups: []string{""},
 				Resources: []string{"pods"},
 			},

mysqlcluster/syncer/status.go

@@ -19,6 +19,7 @@ package syncer
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"time"
 
 	"github.com/presslabs/controller-util/syncer"
@@ -102,8 +103,8 @@ func (s *StatusSyncer) Sync(ctx context.Context) (syncer.SyncResult, error) {
 	// get ready nodes.
 	var readyNodes []corev1.Pod
 	for _, pod := range list.Items {
-		if pod.ObjectMeta.Labels[utils.LableRebuild] == "true" {
-			if err := s.AutoRebuild(ctx, &pod); err != nil {
+		if len(pod.ObjectMeta.Labels[utils.LableRebuild]) > 0 {
+			if err := s.AutoRebuild(ctx, &pod, list.Items); err != nil {
 				s.log.Error(err, "failed to AutoRebuild", "pod", pod.Name, "namespace", pod.Namespace)
 			}
 			continue
@@ -199,12 +200,32 @@ func (s *StatusSyncer) updateClusterStatus() apiv1alpha1.ClusterCondition {
 // Rebuild Pod by deleting and creating it.
 // Notice: This function just delete Pod and PVC,
 // then after k8s recreate pod, it will clone and initial it.
-func (s *StatusSyncer) AutoRebuild(ctx context.Context, pod *corev1.Pod) error {
+func (s *StatusSyncer) AutoRebuild(ctx context.Context, pod *corev1.Pod, items []corev1.Pod) error {
 	ordinal, err := utils.GetOrdinal(pod.Name)
 	if err != nil {
 		return err
 
 	}
+	if pod.ObjectMeta.Labels[utils.LableRebuild] != "true" {
+		podNumber, err := strconv.Atoi(pod.ObjectMeta.Labels[utils.LableRebuild])
+		if err != nil {
+			return fmt.Errorf("rebuild label should be true, or number")
+		}
+		for _, other := range items {
+			ord, err2 := utils.GetOrdinal(other.Name)
+			if err2 != nil {
+				return err
+
+			}
+			if ord == podNumber {
+				other.Labels[utils.LabelRebuildFrom] = "true"
+				if err := s.cli.Update(ctx, &other); err != nil {
+					return err
+				}
+				break
+			}
+		}
+	}
 	// Set Pod UnHealthy.
 	pod.Labels["healthy"] = "no"
 	if err := s.cli.Update(ctx, pod); err != nil {

sidecar/init.go

@@ -17,6 +17,7 @@ limitations under the License.
 package sidecar
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -30,6 +31,11 @@ import (
 	"github.com/go-ini/ini"
 	"github.com/radondb/radondb-mysql-kubernetes/utils"
 	"github.com/spf13/cobra"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 )
 
 // NewInitCommand return a pointer to cobra.Command.
@@ -82,6 +88,13 @@ func CheckServiceExist(cfg *Config, service string) bool {
 func runCloneAndInit(cfg *Config) error {
 	//check follower is exists?
 	serviceURL := ""
+	// Check the rebuildFrom exist?
+	if service, err := getPod(cfg); err == nil {
+		serviceURL = service
+		log.Info("found the rebuild-from pod", "service", serviceURL)
+	} else {
+		log.Info("found the rebuild-from pod", "error", err.Error())
+	}
 	if len(serviceURL) == 0 && CheckServiceExist(cfg, "follower") {
 		serviceURL = fmt.Sprintf("http://%s-%s:%v", cfg.ClusterName, "follower", utils.XBackupPort)
 	}
@@ -339,3 +352,43 @@ func buildSSLdata() error {
 	}
 	return nil
 }
+
+func getPod(cfg *Config) (string, error) {
+	log.Info("Now check the pod which has got rebuild-from")
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return "", err
+	}
+	// creates the clientset
+	clientset, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return "", err
+	}
+	match := map[string]string{
+		utils.LabelRebuildFrom: "true",
+	}
+	podList, err := clientset.CoreV1().Pods(cfg.NameSpace).
+		List(context.TODO(), v1.ListOptions{
+			LabelSelector: labels.SelectorFromSet(match).String()})
+	if err != nil {
+		return "", err
+	}
+	if len(podList.Items) == 1 {
+		pod := podList.Items[0]
+		// Patch remove rebuild-from
+		if err := removeRebuildFrom(clientset, cfg, pod.Name); err != nil {
+			log.Info("remove rebuild from", "error", err.Error())
+		}
+		return fmt.Sprintf("%s.%s-mysql.%s:%v", pod.Name, cfg.ClusterName, cfg.NameSpace, utils.XBackupPort), nil
+	} else {
+		return "", fmt.Errorf("not correct pod choose")
+	}
+
+}
+
+func removeRebuildFrom(clientset *kubernetes.Clientset, cfg *Config, podName string) error {
+	patch := fmt.Sprintf(`[{"op": "remove", "path": "/metadata/labels/%s"}]`, utils.LabelRebuildFrom)
+	_, err := clientset.CoreV1().Pods(cfg.NameSpace).Patch(context.TODO(), podName, types.JSONPatchType, []byte(patch), v1.PatchOptions{})
+
+	return err
+}

utils/constants.go

@@ -195,6 +195,7 @@ const (
 )
 
 const LableRebuild = "rebuild"
+const LabelRebuildFrom = "rebuild-from"
 
 // XenonHttpUrl is a http url corresponding to the xenon instruction.
 type XenonHttpUrl string