Create Scheduler Charts (#12)

* initial charts. need to fix caching issue to confirm Signed-off-by: Cassandra Coyle <[email protected]> * Adds ETCD data dir option, use emptyDir in statefulset Signed-off-by: joshvanl <[email protected]> * fix healthz port Signed-off-by: Cassandra Coyle <[email protected]> * updates to charts Signed-off-by: Cassandra Coyle <[email protected]> * wip. need to fix connectivity issue, but added to injector and updated client code to be an abstract wrapper Signed-off-by: Cassandra Coyle <[email protected]> * Make scheduler host address singular Signed-off-by: joshvanl <[email protected]> * Update cmd/scheduler/options/options.go Co-authored-by: Cassie Coyle <[email protected]> Signed-off-by: Josh van Leeuwen <[email protected]> * rm reminders service name condition Signed-off-by: Cassandra Coyle <[email protected]> * add listen address for scheduler Signed-off-by: Cassandra Coyle <[email protected]> * rebase in trigger reminders via scheduler PR and fix actors scheduler client Signed-off-by: Cassandra Coyle <[email protected]> * ha -> replicaCount, update conditionals including it and use Etcd over ETCD vars Signed-off-by: Cassandra Coyle <[email protected]> * rm volumeClaimTemplates and cleanup Signed-off-by: Cassandra Coyle <[email protected]> * fix version issue Signed-off-by: Cassandra Coyle <[email protected]> * rm diff Signed-off-by: Cassandra Coyle <[email protected]> * updates to ensure ha for k8s works Signed-off-by: Cassandra Coyle <[email protected]> * condense var into 1 line Signed-off-by: Cassandra Coyle <[email protected]> --------- Signed-off-by: Cassandra Coyle <[email protected]> Signed-off-by: joshvanl <[email protected]> Signed-off-by: Josh van Leeuwen <[email protected]> Co-authored-by: joshvanl <[email protected]>
cicoyle · May 24, 2024 · d429070 · d429070
1 parent d4e32cf
commit d429070
Show file tree

Hide file tree

Showing 25 changed files with 589 additions and 372 deletions.
diff --git a/Makefile b/Makefile
@@ -105,7 +105,7 @@ else
 	RUN_BUILD_TOOLS ?= cd .build-tools; GOOS=$(TARGET_OS_LOCAL) GOARCH=$(TARGET_ARCH_LOCAL) go run .
 endif
 
-# Default docker container and e2e test targst.
+# Default docker container and e2e test target.
 TARGET_OS ?= linux
 TARGET_ARCH ?= amd64
 TEST_OUTPUT_FILE_PREFIX ?= ./test_report
@@ -275,7 +275,8 @@ ifeq ($(ONLY_DAPR_IMAGE),true)
 		--set dapr_placement.image.name=$(RELEASE_NAME) \
 		--set dapr_sentry.image.name=$(RELEASE_NAME) \
 		--set dapr_sidecar_injector.image.name=$(RELEASE_NAME) \
-		--set dapr_sidecar_injector.injectorImage.name=$(RELEASE_NAME)
+		--set dapr_sidecar_injector.injectorImage.name=$(RELEASE_NAME) \
+		--set dapr_scheduler.image.name=$(RELEASE_NAME)
 endif
 docker-deploy-k8s: check-docker-env check-arch
 	$(info Deploying ${DAPR_REGISTRY}/${RELEASE_NAME}:${DAPR_TAG} to the current K8S context...)
@@ -349,6 +350,7 @@ TEST_WITH_RACE=./pkg/acl/... \
 ./pkg/retry/... \
 ./pkg/resiliency/... \
 ./pkg/runtime/... \
+./pkg/scheduler/... \
 ./pkg/scopes/... \
 ./pkg/security/... \
 ./pkg/sentry/... \

diff --git a/charts/dapr/Chart.yaml b/charts/dapr/Chart.yaml
@@ -19,3 +19,6 @@ dependencies:
   - name: dapr_sentry
     version: '0.0.0'
     repository: "file://dapr_sentry"
+  - name: dapr_scheduler
+    version: '0.0.0'
+    repository: "file://dapr_scheduler"
diff --git a/charts/dapr/README.md b/charts/dapr/README.md
diff --git a/charts/dapr/charts/dapr_rbac/templates/scheduler.yaml b/charts/dapr/charts/dapr_rbac/templates/scheduler.yaml
@@ -0,0 +1,54 @@
+{{- if or (eq .Values.global.scheduler.enabled true) (eq .Values.global.reminders.serviceName "scheduler") }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: dapr-scheduler
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- range $key, $value := .Values.global.k8sLabels }}
+    {{ $key }}: {{ tpl $value $ }}
+    {{- end }}
+{{- end }}
+---
+{{- if or (eq .Values.global.scheduler.enabled true) (eq .Values.global.reminders.serviceName "scheduler") }}
+{{- if eq .Values.global.rbac.namespaced true }}
+kind: Role
+{{- else }}
+kind: ClusterRole
+{{- end }}
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: dapr-scheduler
+  labels:
+    {{- range $key, $value := .Values.global.k8sLabels }}
+    {{ $key }}: {{ tpl $value $ }}
+    {{- end }}
+rules: []
+{{- end }}
+---
+{{- if or (eq .Values.global.scheduler.enabled true) (eq .Values.global.reminders.serviceName "scheduler") }}
+{{- if eq .Values.global.rbac.namespaced true }}
+kind: RoleBinding
+{{- else }}
+kind: ClusterRoleBinding
+{{- end }}
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: dapr-scheduler
+  labels:
+    {{- range $key, $value := .Values.global.k8sLabels }}
+    {{ $key }}: {{ tpl $value $ }}
+    {{- end }}
+subjects:
+  - kind: ServiceAccount
+    name: dapr-scheduler
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+{{- if eq .Values.global.rbac.namespaced true }}
+  kind: Role
+{{- else }}
+  kind: ClusterRole
+{{- end }}
+  name: dapr-scheduler
+{{- end }}
diff --git a/charts/dapr/charts/dapr_scheduler/templates/NOTES.txt b/charts/dapr/charts/dapr_scheduler/templates/NOTES.txt
diff --git a/charts/dapr/charts/dapr_scheduler/templates/_helpers.tpl b/charts/dapr/charts/dapr_scheduler/templates/_helpers.tpl
@@ -2,61 +2,64 @@
 Expand the name of the chart.
 */}}
 {{- define "dapr_scheduler.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
-{{- end }}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
 
 {{/*
 Create a default fully qualified app name.
 We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
 If release name contains chart name it will be used as a full name.
 */}}
 {{- define "dapr_scheduler.fullname" -}}
-{{- if .Values.fullnameOverride }}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- $name := default .Chart.Name .Values.nameOverride }}
-{{- if contains $name .Release.Name }}
-{{- .Release.Name | trunc 63 | trimSuffix "-" }}
-{{- else }}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
-{{- end }}
-{{- end }}
-{{- end }}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
 
 {{/*
-Create chart name and version as used by the chart label.
+Create initial cluster peer list dynamically based on replicaCount.
 */}}
-{{- define "dapr_scheduler.chart" -}}
-{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
-{{- end }}
+{{- define "dapr_scheduler.initialcluster" -}}
+{{- $initialCluster := "" -}}
+{{- $namespace := .Release.Namespace -}}
+{{- $replicaCount := int .Values.replicaCount -}}
+{{- range $i, $e := until $replicaCount -}}
+{{- $instanceName := printf "dapr-scheduler-server-%d" $i -}}
+{{- $svcName := printf "%s.dapr-scheduler-server.%s.svc.cluster.local" $instanceName $namespace -}}
+{{- $peer := printf "%s=http://%s:%d" $instanceName $svcName (int $.Values.ports.etcdRPCPeerPort) -}}
+{{- $initialCluster = printf "%s%s" $initialCluster $peer -}}
+{{- if ne (int $i) (sub $replicaCount 1) -}}
+{{- $initialCluster = printf "%s," $initialCluster -}}
+{{- end -}}
+{{- end -}}
+{{- $initialCluster -}}
+{{- end -}}
 
 {{/*
-Common labels
+Create etcd client ports list dynamically based on replicaCount.
 */}}
-{{- define "dapr_scheduler.labels" -}}
-helm.sh/chart: {{ include "dapr_scheduler.chart" . }}
-{{ include "dapr_scheduler.selectorLabels" . }}
-{{- if .Chart.AppVersion }}
-app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
-{{- end }}
-app.kubernetes.io/managed-by: {{ .Release.Service }}
-{{- end }}
+{{- define "dapr_scheduler.etcdclientports" -}}
+{{- $etcdClientPorts := "" -}}
+{{- $namespace := .Release.Namespace -}}
+{{- $replicaCount := int .Values.replicaCount -}}
+{{- range $i, $e := until $replicaCount -}}
+{{- $instanceName := printf "dapr-scheduler-server-%d" $i -}}
+{{/*{{- $svcName := printf "%s.%s" $instanceName $namespace -}}*/}}
+{{- $clientPort := int $.Values.ports.etcdRPCClientPort -}}
+{{- $instancePortPair := printf "%s=%d" $instanceName $clientPort -}}
+{{- if gt $i 0 -}}
+{{- $etcdClientPorts = printf "%s,%s" $etcdClientPorts $instancePortPair -}}
+{{- else -}}
+{{- $etcdClientPorts = $instancePortPair -}}
+{{- end -}}
+{{- end -}}
+{{- $etcdClientPorts -}}
+{{- end -}}
 
-{{/*
-Selector labels
-*/}}
-{{- define "dapr_scheduler.selectorLabels" -}}
-app.kubernetes.io/name: {{ include "dapr_scheduler.name" . }}
-app.kubernetes.io/instance: {{ .Release.Name }}
-{{- end }}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "dapr_scheduler.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create }}
-{{- default (include "dapr_scheduler.fullname" .) .Values.serviceAccount.name }}
-{{- else }}
-{{- default "default" .Values.serviceAccount.name }}
-{{- end }}
-{{- end }}
diff --git a/charts/dapr/charts/dapr_scheduler/templates/dapr_scheduler_poddisruptionbudget.yaml b/charts/dapr/charts/dapr_scheduler/templates/dapr_scheduler_poddisruptionbudget.yaml
@@ -0,0 +1,32 @@
+{{- if and (gt (int .Values.replicaCount) 1) (eq .Values.global.scheduler.enabled true) }}
+{{- if .Capabilities.APIVersions.Has "policy/v1" }}
+apiVersion: policy/v1
+{{- else }}
+apiVersion: policy/v1beta1
+{{- end }}
+kind: PodDisruptionBudget
+metadata:
+  name: dapr-scheduler-server-disruption-budget
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: dapr-scheduler-server
+    {{- range $key, $value := .Values.global.k8sLabels }}
+    {{ $key }}: {{ tpl $value $ }}
+    {{- end }}
+spec:
+{{- if .Values.global.ha.disruption.minimumAvailable }}
+  minAvailable: {{ .Values.global.ha.disruption.minimumAvailable }}
+{{- end }}
+{{- if .Values.global.ha.disruption.maximumUnavailable }}
+  maxUnavailable: {{ .Values.global.ha.disruption.maximumUnavailable }}
+{{- end }}
+  selector:
+    matchLabels:
+      app: dapr-scheduler-server
+      {{- range $key, $value := .Values.global.k8sLabels }}
+      {{ $key }}: {{ tpl $value $ }}
+      {{- end }}
+      {{- with .Values.global.labels }}
+        {{- toYaml . | nindent 6 }}
+      {{- end }}
+{{- end }}
diff --git a/charts/dapr/charts/dapr_scheduler/templates/dapr_scheduler_service.yaml b/charts/dapr/charts/dapr_scheduler/templates/dapr_scheduler_service.yaml
@@ -0,0 +1,29 @@
+{{- if (eq .Values.global.scheduler.enabled true) }}
+kind: Service
+apiVersion: v1
+metadata:
+  name: dapr-scheduler-server
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: dapr-scheduler-server
+    {{- range $key, $value := .Values.global.k8sLabels }}
+    {{ $key }}: {{ tpl $value $ }}
+    {{- end }}
+{{- if .Values.service.annotations }}
+{{ toYaml .Values.service.annotations | indent 4}}
+{{- end }}
+spec:
+  selector:
+    app: dapr-scheduler-server
+  # scheduler must be able to resolve pod address to join initial cluster peers
+  # before POD is ready
+  publishNotReadyAddresses: true
+  ports:
+  - name: api
+    port: {{ .Values.ports.apiPort }}
+  - name: etcd-client
+    port: {{ .Values.ports.etcdRPCClientPort }}
+  - name: etcd-peer
+    port: {{ .Values.ports.etcdRPCPeerPort }}
+  clusterIP: None # make the service headless
+{{- end }}