Ms sqlbackupjob (#288)

* more verbose logging on oidc * first pass * working one time job testing 2 week cronjob * changed cron timing * fixed description * created mediocre backup job and clean up job * fixed line wrap for jherts sanity * added cron jobs to be applied only in prod
DataBiosphere · Sep 23, 2019 · 0c0ab8e · 0c0ab8e
1 parent ca7a574
commit 0c0ab8e
Show file tree

Hide file tree

Showing 4 changed files with 99 additions and 0 deletions.
diff --git a/ops/deploy.sh b/ops/deploy.sh
@@ -96,6 +96,12 @@ kubectl --namespace="${KUBE_NAMESPACE}" apply -f "${SCRATCH}/ops/k8s/services"
 # create Deployments
 kubectl --namespace="${KUBE_NAMESPACE}" apply -f "${SCRATCH}/ops/k8s/deployments"
 
+# sql cronjobs for prod
+if [ ${ENVIRONMENT} == "prod" ]
+then
+kubectl --namespace="${KUBE_NAMESPACE}" apply -f "${SCRATCH}/ops/k8s/jobs"
+fi
+
 # build a docker container and push it to gcr
 pushd ${WD}/..
 GCR_TAG=$DATA_REPO_TAG ./gradlew jib

diff --git a/ops/k8s/configs/sql-backup-sa.yaml.ctmpl b/ops/k8s/configs/sql-backup-sa.yaml.ctmpl
@@ -0,0 +1,13 @@
+{{ with $environment := env "ENVIRONMENT"}}
+{{ with $suffix := env "SUFFIX" }}
+{{ with $sa_secret := secret (printf "secret/dsde/datarepo/%s/sql-backup-sa-%s.json" $environment $suffix)}}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: sql-backup-sa
+type: Opaque
+data:
+  credential-file-json: {{$sa_secret.Data | toJSON | base64Encode}}
+{{end}}
+{{end}}
+{{end}}
diff --git a/ops/k8s/jobs/sqlbackup-cleanupjob.yaml.ctmpl b/ops/k8s/jobs/sqlbackup-cleanupjob.yaml.ctmpl
@@ -0,0 +1,41 @@
+{{ with $environment := env "ENVIRONMENT"}}
+{{ with $suffix := env "SUFFIX" }}
+{{ with $sa_secret := secret (printf "secret/dsde/datarepo/%s/sql-backup-sa-%s.json" $environment $suffix)}}
+{{ with $api_secret := secret (printf "secret/dsde/datarepo/%s/api-secrets-%s.json" $environment $suffix)}}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: sqlbackup-cleanupjob
+spec:
+  schedule: "0 1 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: jade-sa
+          containers:
+          - name: sqlbackup-cleanup
+            image: google/cloud-sdk:slim
+            command: ["/bin/sh", "-c", "gcloud auth activate-service-account {{$sa_secret.Data.client_email}} \
+            --key-file=/secrets/key.json && apt-get -qq install jq -y > /dev/null && export removaldate=\"$(date \'+%Y%m%d\' -d \"+90 days ago\")\" \
+            && backups=$(gcloud sql backups list --instance {{$api_secret.Data.instanceName}} --project {{$sa_secret.Data.project_id}} \
+            --format json | jq  -r \'.[]| select (.| has(\"description\"))| select(.description < env.removaldate)\') && \
+            for i in $(echo \"${backups}\" | jq -r \'.id\'); do gcloud sql backups delete $i --instance {{$api_secret.Data.instanceName}} \
+            --project {{$sa_secret.Data.project_id}} --quiet; done"]
+            volumeMounts:
+              - name: sql-backup-credential
+                mountPath: /secrets/
+                readOnly: true
+          volumes:
+            - name: sql-backup-credential
+              secret:
+                secretName: sql-backup-sa
+                items:
+                  - key: credential-file-json
+                    path: key.json
+          restartPolicy: Never
+      backoffLimit: 4
+{{end}}
+{{end}}
+{{end}}
+{{end}}
diff --git a/ops/k8s/jobs/sqlbackup-cronjob.yaml.ctmpl b/ops/k8s/jobs/sqlbackup-cronjob.yaml.ctmpl
@@ -0,0 +1,39 @@
+{{ with $environment := env "ENVIRONMENT"}}
+{{ with $suffix := env "SUFFIX" }}
+{{ with $sa_secret := secret (printf "secret/dsde/datarepo/%s/sql-backup-sa-%s.json" $environment $suffix)}}
+{{ with $api_secret := secret (printf "secret/dsde/datarepo/%s/api-secrets-%s.json" $environment $suffix)}}
+apiVersion: batch/v1beta1
+kind: CronJob
+metadata:
+  name: sqlbackupjob-cronjob
+spec:
+  schedule: "0 3 * * *"
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: jade-sa
+          containers:
+          - name: sqlbackup
+            image: google/cloud-sdk:slim
+            command: ["/bin/sh", "-c", "gcloud auth activate-service-account {{$sa_secret.Data.client_email}} \
+             --key-file=/secrets/key.json && gcloud sql backups create --async --instance \
+             {{$api_secret.Data.instanceName}} --description \"$(date '+%Y%m%d')\" --project \
+             {{$sa_secret.Data.project_id}}"]
+            volumeMounts:
+              - name: sql-backup-credential
+                mountPath: /secrets/
+                readOnly: true
+          volumes:
+            - name: sql-backup-credential
+              secret:
+                secretName: sql-backup-sa
+                items:
+                  - key: credential-file-json
+                    path: key.json
+          restartPolicy: Never
+      backoffLimit: 4
+{{end}}
+{{end}}
+{{end}}
+{{end}}