put trd report file in a bucket for analysis (#53)

* put trd report file in a bucket for analysis it makes it easy to do development work on TRD and gives users an easy way to access detailed csv payment files bucket names are randomized, and we use workload identity to give bucket write rights to containers. * do not login to kubectl, recursive upload
midl-dev · Dec 30, 2021 · 47492c3 · 47492c3
1 parent c67c24f
commit 47492c3
Show file tree

Hide file tree

Showing 7 changed files with 109 additions and 2 deletions.
diff --git a/docker/payout-report-uploader/Dockerfile.template b/docker/payout-report-uploader/Dockerfile.template
@@ -0,0 +1,8 @@
+FROM google/cloud-sdk:slim
+
+COPY entrypoint.sh /payout-report-uploader/
+
+ENTRYPOINT ["/payout-report-uploader/entrypoint.sh"]
+
+CMD []
+
diff --git a/docker/payout-report-uploader/entrypoint.sh b/docker/payout-report-uploader/entrypoint.sh
@@ -0,0 +1,10 @@
+#!/bin/bash -x
+
+find /app/reports
+
+echo "now rsyncing payout reports to $REPORT_BUCKET_URL"
+# workload identity allows this to work
+gsutil -m rsync -r /app/reports $REPORT_BUCKET_URL
+
+echo "Done"
+echo ""
diff --git a/k8s/payout-base/trd-payout.yaml b/k8s/payout-base/trd-payout.yaml
@@ -24,7 +24,7 @@ spec:
         spec:
           securityContext:
             fsGroup: 100
-          containers:
+          initContainers:
           - name: trd-payout-cron
             image: trd
               #command: ["/bin/sh", "-ec", "sleep 1000"]
@@ -38,6 +38,20 @@ spec:
             resources:
               limits:
                 cpu: 0
+          containers:
+          - name: report-uploader
+            image: payout-report-uploader
+            volumeMounts:
+            - name: trd-reports
+              mountPath: /app/reports
+              readOnly: true
+            imagePullPolicy: Always
+            resources:
+              limits:
+                cpu: 0
+            envFrom:
+            - configMapRef:
+                name: report-upload-config
           volumes:
           - name: trd-reports
             persistentVolumeClaim:
@@ -49,6 +63,7 @@ spec:
               - key: config.yaml
                 path: config.yaml
           restartPolicy: OnFailure
+          serviceAccountName: trd-report-uploader
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
@@ -68,3 +83,8 @@ spec:
     - podSelector:
         matchLabels:
           app: trd-payout-sender
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: trd-report-uploader
diff --git a/k8s/payout-tmpl/kustomization.yaml.tmpl b/k8s/payout-tmpl/kustomization.yaml.tmpl
@@ -10,13 +10,21 @@ imageTags:
   - name: trd
     newName: registry.gitlab.com/ochem/tezos-reward-distributor
     newTag: latest
+  - name: payout-report-uploader
+    newName: gcr.io/${project}/payout-report-uploader
+    newTag: ${kubernetes_namespace}-latest
 
 configMapGenerator:
 - name: trd-config
   files:
   - config.yaml
+- name: report-upload-config
+  literals:
+  - GCP_REGION="${region}"
+  - REPORT_BUCKET_URL="${report_bucket_url}"
 
 patchesStrategicMerge:
 - crontime.yaml
 - nodepool.yaml
 - trd-args.yaml
+- serviceaccountannotate.yaml
diff --git a/k8s/payout-tmpl/serviceaccountannotate.yaml.tmpl b/k8s/payout-tmpl/serviceaccountannotate.yaml.tmpl
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: trd-report-uploader
+  annotations:
+    iam.gke.io/gcp-service-account: ${kubernetes_name_prefix}-payout-report-uploader@${project}.iam.gserviceaccount.com
diff --git a/k8s/payout-tmpl/trd-args.yaml.tmpl b/k8s/payout-tmpl/trd-args.yaml.tmpl
@@ -7,7 +7,7 @@ spec:
     spec:
       template:
         spec:
-          containers:
+          initContainers:
           - name: trd-payout-cron
             args:
             - "--run_mode"

diff --git a/terraform/k8s.tf b/terraform/k8s.tf
@@ -199,6 +199,8 @@ ${templatefile("${path.module}/../k8s/payout-tmpl/kustomization.yaml.tmpl",
   merge(var.baking_nodes[nodename][baker_name]["payout_config"], {
   "project": var.project,
   "baker_name": baker_name,
+  "region": module.terraform-gke-blockchain.location,
+  "report_bucket_url": google_storage_bucket.trd_report_bucket[baker_name].url
   "kubernetes_name_prefix": var.kubernetes_name_prefix,
   "kubernetes_namespace": var.kubernetes_namespace} ))}
 EOK
@@ -227,6 +229,10 @@ EOA
 cat <<EOPC > payout-${baker_name}/crontime.yaml
 ${templatefile("${path.module}/../k8s/payout-tmpl/crontime.yaml.tmpl", {"schedule": var.baking_nodes[nodename][baker_name]["payout_config"]["schedule"]})}
 EOPC
+echo Now writing to payout-${baker_name}/serviceaccountannotate.yaml
+cat <<EONPN > payout-${baker_name}/serviceaccountannotate.yaml
+${templatefile("${path.module}/../k8s/payout-tmpl/serviceaccountannotate.yaml.tmpl", local.kubernetes_variables)}
+EONPN
 %{ endif }
 
 %{ endfor}
@@ -289,3 +295,52 @@ resource "google_compute_security_policy" "public_rpc_filter" {
   }
 
 }
+
+#############################
+# Reward Buckets
+#############################
+
+# we create buckets for every baker (even if they are not public)
+# because it's simpler
+
+resource "google_service_account" "payout_report_uploader" {
+  account_id   = "${var.kubernetes_name_prefix}-payout-report-uploader"
+  display_name = "Payout report uploader for ${var.kubernetes_name_prefix}"
+  project = module.terraform-gke-blockchain.project 
+}
+
+# based on workload identity docs
+# https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity
+resource "google_service_account_iam_binding" "payout_report_uploader_binding" {
+  service_account_id = google_service_account.payout_report_uploader.name
+  role               = "roles/iam.workloadIdentityUser"
+
+  members = [ for k in toset(keys(merge(merge(values(var.baking_nodes)...),{}))) :
+    "serviceAccount:${module.terraform-gke-blockchain.project}.svc.id.goog[${var.kubernetes_namespace}/${var.kubernetes_name_prefix}-trd-report-uploader-${k}]"
+  ]
+}
+resource "random_id" "rnd_bucket" {
+  for_each = toset(keys(merge(merge(values(var.baking_nodes)...),{})))
+  byte_length = 4
+}
+resource "google_storage_bucket" "trd_report_bucket" {
+  for_each = toset(keys(merge(merge(values(var.baking_nodes)...),{})))
+  name     = "${var.kubernetes_name_prefix}-baker-payout-${each.key}-${random_id.rnd_bucket[each.key].hex}"
+  project = module.terraform-gke-blockchain.project
+
+  force_destroy = true
+}
+
+resource "google_storage_bucket_iam_member" "member" {
+  for_each = toset(keys(merge(merge(values(var.baking_nodes)...),{})))
+  bucket = google_storage_bucket.trd_report_bucket[each.key].name
+  role        = "roles/storage.objectAdmin"
+  member      = "serviceAccount:${google_service_account.payout_report_uploader.email}"
+}
+
+resource "google_storage_bucket_iam_member" "make_public" {
+  for_each = toset(keys(merge(merge(values(var.baking_nodes)...),{})))
+  bucket = google_storage_bucket.trd_report_bucket[each.key].name
+  role        = "roles/storage.objectViewer"
+  member      = "allUsers"
+}