From 6e811fa31e998f219616b9f0324a705931947169 Mon Sep 17 00:00:00 2001 From: nobolity Date: Tue, 27 Sep 2022 12:18:19 +0800 Subject: [PATCH] Add serviceaccount for DS components to read configmaps (#4) --- api/v1alpha1/ds_public.go | 3 + api/v1alpha1/dsalert_types.go | 2 + api/v1alpha1/dsapi_types.go | 2 + api/v1alpha1/dsmaster_types.go | 2 + api/v1alpha1/dsworker_types.go | 2 + ....apache.dolphinscheduler.dev_dsalerts.yaml | 2 + ...ds.apache.dolphinscheduler.dev_dsapis.yaml | 2 + ...apache.dolphinscheduler.dev_dsmasters.yaml | 2 + ...apache.dolphinscheduler.dev_dsworkers.yaml | 2 + config/manager/kustomization.yaml | 2 +- config/rbac/role.yaml | 27 +++++++ config/samples/ds_v1alpha1_dsalert.yaml | 5 +- config/samples/ds_v1alpha1_dsapi.yaml | 5 +- config/samples/ds_v1alpha1_dsmaster.yaml | 5 +- config/samples/ds_v1alpha1_dsworker.yaml | 14 +--- controllers/alert_reconcile.go | 1 + controllers/api_reconcile.go | 1 + controllers/dsmaster_controller.go | 79 +++++++++++++++++++ controllers/master_reconcile.go | 50 ++++++++++-- controllers/worker_reconcile.go | 7 +- main.go | 12 +-- 21 files changed, 196 insertions(+), 31 deletions(-) diff --git a/api/v1alpha1/ds_public.go b/api/v1alpha1/ds_public.go index 78b2b63..670d884 100644 --- a/api/v1alpha1/ds_public.go +++ b/api/v1alpha1/ds_public.go @@ -66,6 +66,9 @@ const ( DsApiPort = 12345 DsAlertPort = 50052 DsWorkerHpa = "ds-worker-hpa" + DsServiceAccount = "ds-service-account" + DsRole = "ds-role" + DsRoleBinding = "ds-role-binding" ) // DsCondition represents one current condition of a ds cluster. diff --git a/api/v1alpha1/dsalert_types.go b/api/v1alpha1/dsalert_types.go index a57fa1a..acd27cc 100644 --- a/api/v1alpha1/dsalert_types.go +++ b/api/v1alpha1/dsalert_types.go @@ -43,6 +43,8 @@ type DSAlertSpec struct { // +kubebuilder:default=apache/dolphinscheduler-master Repository string `json:"repository,omitempty"` + ServiceAccount string `json:"service_account,omitempty"` + // Replicas is the expected size of the ms-master. // The ds-master-operator will eventually make the size of the running // equal to the expected size. diff --git a/api/v1alpha1/dsapi_types.go b/api/v1alpha1/dsapi_types.go index ca70603..bbcf1bc 100644 --- a/api/v1alpha1/dsapi_types.go +++ b/api/v1alpha1/dsapi_types.go @@ -39,6 +39,8 @@ type DSApiSpec struct { ZookeeperConnect string `json:"zookeeper_connect,omitempty"` + ServiceAccount string `json:"service_account,omitempty"` + // Repository is the name of the repository that hosts // ds container images. It should be direct clone of the repository in official // By default, it is `apache/dolphinscheduler-master`. diff --git a/api/v1alpha1/dsmaster_types.go b/api/v1alpha1/dsmaster_types.go index f7971ca..e3dcd42 100644 --- a/api/v1alpha1/dsmaster_types.go +++ b/api/v1alpha1/dsmaster_types.go @@ -61,6 +61,8 @@ type DSMasterSpec struct { HpaPolicy *HpaPolicy `json:"hpa,omitempty"` + ServiceAccount string `json:"service_account,omitempty"` + // Paused is to pause the control of the operator for the ds-master . // +kubebuilder:default=false Paused bool `json:"paused,omitempty"` diff --git a/api/v1alpha1/dsworker_types.go b/api/v1alpha1/dsworker_types.go index e0dc9b0..81ef041 100644 --- a/api/v1alpha1/dsworker_types.go +++ b/api/v1alpha1/dsworker_types.go @@ -38,6 +38,8 @@ type DSWorkerSpec struct { // +kubebuilder:default="3.0.0-alpha" Version string `json:"version,omitempty"` + ServiceAccount string `json:"service_account,omitempty"` + // Repository is the name of the repository that hosts // ds container images. It should be direct clone of the repository in official // By default, it is `apache/dolphinscheduler-worker`. diff --git a/config/crd/bases/ds.apache.dolphinscheduler.dev_dsalerts.yaml b/config/crd/bases/ds.apache.dolphinscheduler.dev_dsalerts.yaml index 2781983..4efc259 100644 --- a/config/crd/bases/ds.apache.dolphinscheduler.dev_dsalerts.yaml +++ b/config/crd/bases/ds.apache.dolphinscheduler.dev_dsalerts.yaml @@ -1148,6 +1148,8 @@ spec: container images. It should be direct clone of the repository in official By default, it is `apache/dolphinscheduler-master`. type: string + service_account: + type: string version: default: 3.0.0-alpha description: Version is the expected version of the ds cluster. The diff --git a/config/crd/bases/ds.apache.dolphinscheduler.dev_dsapis.yaml b/config/crd/bases/ds.apache.dolphinscheduler.dev_dsapis.yaml index efd8dad..d18656d 100644 --- a/config/crd/bases/ds.apache.dolphinscheduler.dev_dsapis.yaml +++ b/config/crd/bases/ds.apache.dolphinscheduler.dev_dsapis.yaml @@ -1152,6 +1152,8 @@ spec: container images. It should be direct clone of the repository in official By default, it is `apache/dolphinscheduler-master`. type: string + service_account: + type: string version: default: 3.0.0-alpha description: Version is the expected version of the ds cluster. The diff --git a/config/crd/bases/ds.apache.dolphinscheduler.dev_dsmasters.yaml b/config/crd/bases/ds.apache.dolphinscheduler.dev_dsmasters.yaml index 973643f..cf5800b 100644 --- a/config/crd/bases/ds.apache.dolphinscheduler.dev_dsmasters.yaml +++ b/config/crd/bases/ds.apache.dolphinscheduler.dev_dsmasters.yaml @@ -1349,6 +1349,8 @@ spec: container images. It should be direct clone of the repository in official By default, it is `apache/dolphinscheduler-master`. type: string + service_account: + type: string version: default: 3.0.0-alpha description: Version is the expected version of the ds cluster. The diff --git a/config/crd/bases/ds.apache.dolphinscheduler.dev_dsworkers.yaml b/config/crd/bases/ds.apache.dolphinscheduler.dev_dsworkers.yaml index e816f41..00967f7 100644 --- a/config/crd/bases/ds.apache.dolphinscheduler.dev_dsworkers.yaml +++ b/config/crd/bases/ds.apache.dolphinscheduler.dev_dsworkers.yaml @@ -1351,6 +1351,8 @@ spec: container images. It should be direct clone of the repository in official By default, it is `apache/dolphinscheduler-worker`. type: string + service_account: + type: string version: default: 3.0.0-alpha description: Version is the expected version of the ds cluster. The diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 13f8ea0..d69e79a 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -30,4 +30,4 @@ kind: Kustomization images: - name: controller newName: apache/dolphinscheduler-operator - newTag: 0.1.0 + newTag: latest diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 074be47..98d6598 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -44,6 +44,15 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - delete + - get + - list - apiGroups: - "" resources: @@ -184,3 +193,21 @@ rules: - get - patch - update +- apiGroups: + - rbac.authorization.k8s.io + resources: + - role + verbs: + - create + - delete + - get + - list +- apiGroups: + - rbac.authorization.k8s.io + resources: + - rolebinding + verbs: + - create + - delete + - get + - list diff --git a/config/samples/ds_v1alpha1_dsalert.yaml b/config/samples/ds_v1alpha1_dsalert.yaml index 801178d..481f744 100644 --- a/config/samples/ds_v1alpha1_dsalert.yaml +++ b/config/samples/ds_v1alpha1_dsalert.yaml @@ -24,8 +24,9 @@ metadata: app: ds-alert spec: replicas: 1 - version: 3.0.0-alpha - repository: apache/dolphinscheduler-alert-server + version: latest + repository: ghcr.io/apache/dolphinscheduler/dolphinscheduler-alert-server + service_account: "ds-service-account" datasource: drive_name: "org.postgresql.Driver" url: "jdbc:postgresql://postgres-service:5432/dolphinscheduler" diff --git a/config/samples/ds_v1alpha1_dsapi.yaml b/config/samples/ds_v1alpha1_dsapi.yaml index 4d18eaa..7c646bf 100644 --- a/config/samples/ds_v1alpha1_dsapi.yaml +++ b/config/samples/ds_v1alpha1_dsapi.yaml @@ -24,10 +24,11 @@ metadata: app: ds-api spec: replicas: 1 - version: 3.0.0-alpha + version: latest zookeeper_connect: "zookeeper-service:2181" - repository: apache/dolphinscheduler-api + repository: ghcr.io/apache/dolphinscheduler/dolphinscheduler-api node_port: 30002 + service_account: "ds-service-account" datasource: drive_name: "org.postgresql.Driver" url: "jdbc:postgresql://postgres-service:5432/dolphinscheduler" diff --git a/config/samples/ds_v1alpha1_dsmaster.yaml b/config/samples/ds_v1alpha1_dsmaster.yaml index 55aea38..59501ac 100644 --- a/config/samples/ds_v1alpha1_dsmaster.yaml +++ b/config/samples/ds_v1alpha1_dsmaster.yaml @@ -25,8 +25,9 @@ metadata: spec: replicas: 1 zookeeper_connect: "zookeeper-service:2181" - version: 3.0.0-alpha - repository: apache/dolphinscheduler-master + version: latest + repository: ghcr.io/apache/dolphinscheduler/dolphinscheduler-master + service_account: "ds-service-account" datasource: drive_name: "org.postgresql.Driver" url: "jdbc:postgresql://postgres-service:5432/dolphinscheduler" diff --git a/config/samples/ds_v1alpha1_dsworker.yaml b/config/samples/ds_v1alpha1_dsworker.yaml index c0eb982..48a3299 100644 --- a/config/samples/ds_v1alpha1_dsworker.yaml +++ b/config/samples/ds_v1alpha1_dsworker.yaml @@ -25,18 +25,12 @@ metadata: spec: replicas: 1 zookeeper_connect: "zookeeper-service:2181" - version: 3.0.0-alpha - repository: apache/dolphinscheduler-worker + version: latest + repository: ghcr.io/apache/dolphinscheduler/dolphinscheduler-worker + service_account: "ds-service-account" datasource: drive_name: "org.postgresql.Driver" url: "jdbc:postgresql://postgres-service:5432/dolphinscheduler" username: "postgresadmin" password: "admin12345" - pod: - resources: - limits: - cpu: "1000m" - memory: "2Gi" - requests: - cpu: "500m" - memory: "1Gi" + diff --git a/controllers/alert_reconcile.go b/controllers/alert_reconcile.go index b5e1ee2..878e9ce 100644 --- a/controllers/alert_reconcile.go +++ b/controllers/alert_reconcile.go @@ -69,6 +69,7 @@ func createAlertDeployment(cluster *dsv1alpha1.DSAlert) *v1.Deployment { }, }, Spec: corev1.PodSpec{ + ServiceAccountName: cluster.Spec.ServiceAccount, Containers: []corev1.Container{{ Name: dsv1alpha1.DsAlert, Image: ImageName(cluster.Spec.Repository, cluster.Spec.Version), diff --git a/controllers/api_reconcile.go b/controllers/api_reconcile.go index bd5cf09..01ad35c 100644 --- a/controllers/api_reconcile.go +++ b/controllers/api_reconcile.go @@ -75,6 +75,7 @@ func createApiDeployment(cluster *dsv1alpha1.DSApi) *v1.Deployment { }, }, Spec: corev1.PodSpec{ + ServiceAccountName: cluster.Spec.ServiceAccount, Containers: []corev1.Container{{ Name: dsv1alpha1.DsApi, Image: ImageName(cluster.Spec.Repository, cluster.Spec.Version), diff --git a/controllers/dsmaster_controller.go b/controllers/dsmaster_controller.go index fadbdeb..c239ac7 100644 --- a/controllers/dsmaster_controller.go +++ b/controllers/dsmaster_controller.go @@ -20,6 +20,7 @@ package controllers import ( "context" "k8s.io/api/autoscaling/v2beta2" + v1 "k8s.io/api/rbac/v1" "time" dsv1alpha1 "dolphinscheduler-operator/api/v1alpha1" @@ -57,6 +58,9 @@ type DSMasterReconciler struct { //+kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;create;delete;list;watch //+kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups="",resources=services,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups="",resources=serviceaccounts,verbs=get;list;create;delete +//+kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=role,verbs=get;list;create;delete +//+kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebinding,verbs=get;list;create;delete // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -77,6 +81,21 @@ func (r *DSMasterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c } desired := cluster.DeepCopy() + sa := &corev1.ServiceAccount{} + saReq := ctrl.Request{ + NamespacedName: types.NamespacedName{ + Namespace: req.Namespace, + Name: dsv1alpha1.DsServiceAccount, + }, + } + err := r.Get(ctx, saReq.NamespacedName, sa) + if apierrors.IsNotFound(err) { + err := r.createServiceAccountIfNotExists(ctx, cluster) + if err != nil { + return ctrl.Result{}, err + } + } + // Handler finalizer // examine DeletionTimestamp to determine if object is under deletion if cluster.ObjectMeta.DeletionTimestamp.IsZero() { @@ -199,6 +218,9 @@ func (r *DSMasterReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Pod{}). Owns(&corev1.Service{}). Owns(&v2beta2.HorizontalPodAutoscaler{}). + Owns(&corev1.ServiceAccount{}). + Owns(&v1.Role{}). + Owns(&v1.RoleBinding{}). // or use WithEventFilter() WithEventFilter(filter). Complete(r) @@ -435,3 +457,60 @@ func (r *DSMasterReconciler) ensureHPA(ctx context.Context, cluster *dsv1alpha1. } return nil } + +// 创建 ServiceAccount +func (r *DSMasterReconciler) createServiceAccountIfNotExists(ctx context.Context, cluster *dsv1alpha1.DSMaster) (err error) { + + masterLogger.Info("start create service account.") + + sa := &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: dsv1alpha1.DsServiceAccount, + Namespace: cluster.Namespace, + }, + } + + err = r.Create(ctx, sa) + if err != nil { + masterLogger.Error(err, "create service account error") + return err + } + // binding the sa + err = controllerutil.SetControllerReference(cluster, sa, r.Scheme) + if err != nil { + masterLogger.Error(err, "sa SetControllerReference error") + return err + } + + ro := &v1.Role{} + namespacedName := types.NamespacedName{Namespace: cluster.Namespace, Name: dsv1alpha1.DsRole} + if err := r.Client.Get(ctx, namespacedName, ro); err != nil { + if apierrors.IsNotFound(err) && !apierrors.IsAlreadyExists(err) { + // Remote may already exist, so we will return err, for the next time, this code will not execute + ro := r.createRole(cluster) + if err := controllerutil.SetControllerReference(cluster, ro, r.Scheme); err != nil { + masterLogger.Info("set controller role error") + return err + } + if err := r.Client.Create(ctx, ro); err != nil { + return err + } + } + } + + rb := &v1.RoleBinding{} + rbNamespacedName := types.NamespacedName{Namespace: cluster.Namespace, Name: dsv1alpha1.DsRoleBinding} + if err := r.Client.Get(ctx, rbNamespacedName, rb); err != nil { + if apierrors.IsNotFound(err) && !apierrors.IsAlreadyExists(err) { + rb := r.createRoleBinding(cluster) + if err := controllerutil.SetControllerReference(cluster, rb, r.Scheme); err != nil { + masterLogger.Info("set controller rolebinding error") + return err + } + if err := r.Client.Create(ctx, rb); err != nil { + return err + } + } + } + return nil +} diff --git a/controllers/master_reconcile.go b/controllers/master_reconcile.go index d51099d..1cab854 100644 --- a/controllers/master_reconcile.go +++ b/controllers/master_reconcile.go @@ -20,6 +20,7 @@ package controllers import ( "context" dsv1alpha1 "dolphinscheduler-operator/api/v1alpha1" + v1 "k8s.io/api/rbac/v1" "k8s.io/api/autoscaling/v2beta2" @@ -67,14 +68,14 @@ func newDSMasterPod(cr *dsv1alpha1.DSMaster) *corev1.Pod { Name: podName, Namespace: cr.Namespace, Labels: map[string]string{dsv1alpha1.DsAppName: dsv1alpha1.DsMasterLabel, - dsv1alpha1.DsVersionLabel: ImageName(cr.Spec.Repository, cr.Spec.Version), + dsv1alpha1.DsVersionLabel: cr.Spec.Version, dsv1alpha1.DsServiceLabel: dsv1alpha1.DsServiceLabelValue}, }, Spec: corev1.PodSpec{ - Hostname: podName, - Subdomain: dsv1alpha1.DsServiceLabelValue, - SetHostnameAsFQDN: &isSetHostnameAsFQDN, - + Hostname: podName, + Subdomain: dsv1alpha1.DsServiceLabelValue, + SetHostnameAsFQDN: &isSetHostnameAsFQDN, + ServiceAccountName: cr.Spec.ServiceAccount, Containers: []corev1.Container{ { Name: cr.Name, @@ -195,3 +196,42 @@ func (r *DSMasterReconciler) deleteHPA(ctx context.Context, hpa *v2beta2.Horizon } return nil } + +func (r *DSMasterReconciler) createRole(cluster *dsv1alpha1.DSMaster) *v1.Role { + role := v1.Role{ + ObjectMeta: metav1.ObjectMeta{ + Name: dsv1alpha1.DsRole, + Namespace: cluster.Namespace, + Labels: map[string]string{dsv1alpha1.DsAppName: dsv1alpha1.DsRole}, + }, + Rules: []v1.PolicyRule{ + { + Verbs: []string{"get", "watch"}, + Resources: []string{"configmaps"}, + APIGroups: []string{""}, + }}, + } + return &role +} + +func (r *DSMasterReconciler) createRoleBinding(cluster *dsv1alpha1.DSMaster) *v1.RoleBinding { + roleBinding := v1.RoleBinding{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: dsv1alpha1.DsRole, + Namespace: cluster.Namespace, + Labels: map[string]string{dsv1alpha1.DsAppName: dsv1alpha1.DsRoleBinding}, + }, + Subjects: []v1.Subject{{ + Kind: "ServiceAccount", + Name: dsv1alpha1.DsServiceAccount, + Namespace: cluster.Namespace, + }}, + RoleRef: v1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "Role", + Name: dsv1alpha1.DsRole, + }, + } + return &roleBinding +} diff --git a/controllers/worker_reconcile.go b/controllers/worker_reconcile.go index a036f3f..f786d6d 100644 --- a/controllers/worker_reconcile.go +++ b/controllers/worker_reconcile.go @@ -62,13 +62,14 @@ func newDSWorkerPod(cr *dsv1alpha1.DSWorker) *corev1.Pod { Name: podName, Namespace: cr.Namespace, Labels: map[string]string{dsv1alpha1.DsAppName: dsv1alpha1.DsWorkerLabel, - dsv1alpha1.DsVersionLabel: ImageName(cr.Spec.Repository, cr.Spec.Version), + dsv1alpha1.DsVersionLabel: cr.Spec.Version, dsv1alpha1.DsServiceLabel: dsv1alpha1.DsServiceLabelValue, }, }, Spec: corev1.PodSpec{ - Hostname: podName, - Subdomain: dsv1alpha1.DsServiceLabelValue, + Hostname: podName, + Subdomain: dsv1alpha1.DsServiceLabelValue, + ServiceAccountName: cr.Spec.ServiceAccount, Containers: []corev1.Container{ { Name: cr.Name, diff --git a/main.go b/main.go index 8eaf1ef..dc83e5a 100644 --- a/main.go +++ b/main.go @@ -107,12 +107,12 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "DSApi") os.Exit(1) } - if os.Getenv("ENABLE_WEBHOOKS") != "false" { - if err = (&dsv1alpha1.DSMaster{}).SetupWebhookWithManager(mgr); err != nil { - setupLog.Error(err, "unable to create webhook", "webhook", "DSMaster") - os.Exit(1) - } - } + //if os.Getenv("ENABLE_WEBHOOKS") != "false" { + // if err = (&dsv1alpha1.DSMaster{}).SetupWebhookWithManager(mgr); err != nil { + // setupLog.Error(err, "unable to create webhook", "webhook", "DSMaster") + // os.Exit(1) + // } + //} //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {