Migrate CI from AWS queues to Google Kubernetes Engine queues (#1333)

* Migrate CI from AWS queues to Google Kubernetes Engine queues

- Add necessary k8s plugin config to the pipeline for use with GKE + Buildkite's agent-stack-k8s helm stack
- Remove docker agent tag filter
- Install bk agent in Dockerfile for use in the code coverage step
- Add comment on source of buildkite-agent installation commands
- Add comment on yq explode usage to the top of pipeline.yml

Author: mstifflin

Diff for: .buildkite/pipeline.yml

@@ -1,93 +1,155 @@
+# `yq 'explode(.)' .buildkite/pipeline.yml` to view expanded anchors/aliases
+container:
+  kubernetes: &kubernetes
+    gitEnvFrom:
+      - secretRef:
+          name: oss-github-ssh-credentials
+    sidecars:
+    - image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-dind:v1
+      volumeMounts:
+        - mountPath: /var/run/
+          name: docker-sock
+      securityContext:
+        privileged: true
+        allowPrivilegeEscalation: true
+    mirrorVolumeMounts: true # CRITICAL: this must be at the same indentation level as sidecars
+    podSpec: &podSpec
+      containers:
+        - &commandContainer
+          image: us-west1-docker.pkg.dev/ci-compute/buildkite-images/buildkite-command-container:v2
+          command:
+          - |-
+            echo "Command step was not overridden."
+            exit 1
+          volumeMounts:
+            - mountPath: /var/run/
+              name: docker-sock
+          resources:
+            requests:
+              cpu: 7500m
+              memory: 28G
+      volumes:
+      - name: docker-sock
+        emptyDir: {}
+
+agents:
+  queue: "buildkite-gcp"
 steps:
   - label: "fossa analyze"
-    agents:
-      queue: "init"
-      docker: "*"
     command: ".buildkite/scripts/fossa.sh"
 
   - label: ":golang: unit-test"
-    agents:
-      queue: "workers"
-      docker: "*"
-    commands:
-      - "make unit_test"
-      - ".buildkite/scripts/gen_coverage_metadata.sh .build/metadata.txt"
     artifact_paths:
       - ".build/*/coverage/*.out"
       - ".build/cover.out"
       - ".build/metadata.txt"
     plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make unit_test
+                  .buildkite/scripts/gen_coverage_metadata.sh .build/metadata.txt
       - docker-compose#v3.0.0:
           run: unit-test
           config: docker/buildkite/docker-compose.yml
 
   - label: ":golangci-lint: validate code is clean"
-    agents:
-      queue: "workers"
-      docker: "*"
-    command: "./scripts/golint.sh"
     artifact_paths: [ ]
     retry:
       automatic:
-        limit: 1
+        limit: 2
     plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  ./scripts/golint.sh
       - docker-compose#v3.0.0:
           run: unit-test
           config: docker/buildkite/docker-compose.yml
 
   - label: ":golang: integration-test-sticky-off"
-    agents:
-      queue: "workers"
-      docker: "*"
-    command: "make integ_test_sticky_off"
     artifact_paths:
       - ".build/*/coverage/*.out"
     retry:
       automatic:
-        limit: 1
+        limit: 2
     plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make integ_test_sticky_off
       - docker-compose#v3.0.0:
           run: integ-test
           config: docker/buildkite/docker-compose.yml
 
   - label: ":golang: integration-test-sticky-on"
-    agents:
-      queue: "workers"
-      docker: "*"
-    command: "make integ_test_sticky_on"
     artifact_paths:
       - ".build/*/coverage/*.out"
     retry:
       automatic:
-        limit: 1
+        limit: 2
     plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make integ_test_sticky_on
       - docker-compose#v3.0.0:
           run: integ-test
           config: docker/buildkite/docker-compose.yml
 
   - label: ":golang: integration-test-grpc-adapter"
-    agents:
-      queue: "workers"
-      docker: "*"
-    command: "make integ_test_grpc"
     artifact_paths:
       - ".build/*/coverage/*.out"
     retry:
       automatic:
-        limit: 1
+        limit: 2
     plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  make integ_test_grpc
       - docker-compose#v3.0.0:
           run: integ-test-grpc
           config: docker/buildkite/docker-compose.yml
 
   - wait
 
   - label: ":golang: code-coverage"
-    agents:
-      queue: "workers"
-      docker: "*"
-    command: ".buildkite/scripts/gocov.sh"
     plugins:
+      - kubernetes:
+          <<: *kubernetes
+          podSpec:
+            <<: *podSpec
+            containers:
+              - <<: *commandContainer
+                command:
+                - |-
+                  .buildkite/scripts/gocov.sh
       - docker-compose#v3.0.0:
           run: coverage-report
           config: docker/buildkite/docker-compose.yml

Diff for: docker/buildkite/Dockerfile

@@ -8,3 +8,13 @@ ADD go.mod go.sum /go/src/go.uber.org/cadence/
 # allow git-status and similar to work
 RUN git config --global --add safe.directory /go/src/go.uber.org/cadence
 RUN GO111MODULE=on go mod download
+
+# Install Buildkite agent
+# https://buildkite.com/docs/agent/v3/ubuntu
+RUN apt-get install -y apt-transport-https dirmngr curl
+RUN curl -fsSL https://keys.openpgp.org/vks/v1/by-fingerprint/32A37959C2FA5C3C99EFBC32A79206696452D198 | \
+  gpg --dearmor -o /usr/share/keyrings/buildkite-agent-archive-keyring.gpg
+RUN echo \
+  "deb [signed-by=/usr/share/keyrings/buildkite-agent-archive-keyring.gpg] https://apt.buildkite.com/buildkite-agent stable main" | \
+  tee /etc/apt/sources.list.d/buildkite-agent.list
+RUN apt-get update && apt-get install -yy --no-install-recommends buildkite-agent

Diff for: docker/buildkite/docker-compose.yml

@@ -116,7 +116,6 @@ services:
       - "GO111MODULE=on"
     volumes:
       - ../../:/go/src/go.uber.org/cadence
-      - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
 
 networks:
   services-network: