Introduce manifest variants

This change updates the config and Makefile to produce multiple variants
of the HNC manifests. The initial list is "default" (same as today),
"default-cm" (same, but configured for cert-manager instead of the
internal cert-controller), and ha (replicated webhooks, using the
internal cert controller).

Also made some updates to the Prow tests to allow for variant testing,
as well as upgrading the ability to test the Prow tests locally.

Tested: verified that default.yaml is the same as the prior
hnc-manager.yaml, apart from some reordered command line args. Verified
that smoke tests work on ha.yaml. Did not do _any_ testing on the
cert-manager variants. Locally ran the prow e2e tests on both the HA and
default configs. Also did an experimental release from my personal repo
and bumped up the timeout since we've added more build targets.

Author: adrianludwin

Makefile

@@ -2,7 +2,19 @@
 .PHONY: release
 
 # If CONFIG is `kind`, various defaults will be optimized for deploying locally to Kind
-CONFIG ?= "default"
+CONFIG ?= default
+
+# Set the Kind name (by default, it's "kind"). If you set this explicitly,
+# CONFIG is automatically set to "kind" as well, overriding any existing
+# setting.
+ifeq ($(CONFIG),kind)
+  KIND ?= "kind"
+else
+  KIND ?= ""
+endif
+ifneq ($(KIND),"")
+  CONFIG = kind
+endif
 
 # The GCP project ID useful to have when performing operations that require one
 # (e.g. release). If you don't have gcloud, all other operations in this
@@ -157,14 +169,21 @@ manifests: controller-gen
 	cd manifests && \
 		touch kustomization.yaml && \
 		${KUSTOMIZE} edit add resource ../config/crd
-	${KUSTOMIZE} build manifests/ -o manifests/hnc-crds.yaml
-	@echo "Building full manifest"
-	rm manifests/kustomization.yaml
-	cd manifests && \
-		touch kustomization.yaml && \
-		${KUSTOMIZE} edit add resource ../config/default && \
-		${KUSTOMIZE} edit set image controller=${HNC_IMG}
-	${KUSTOMIZE} build manifests/ -o manifests/${HNC_IMG_NAME}.yaml
+	${KUSTOMIZE} build manifests/ -o manifests/crds.yaml
+	@cd manifests && \
+		for variant in default-cc default-cm nowebhooks-cc ha-webhooks-cc ; do \
+			echo "Building $${variant} manifest"; \
+			rm kustomization.yaml; \
+			touch kustomization.yaml && \
+			${KUSTOMIZE} edit add resource ../config/variants/$${variant} && \
+			${KUSTOMIZE} edit set image controller=${HNC_IMG}; \
+			${KUSTOMIZE} build . -o ./$${variant}.yaml; \
+		done
+	@echo "Creating alias and summary manifests"
+	@cp manifests/default-cc.yaml manifests/default.yaml
+	@cat manifests/nowebhooks-cc.yaml > manifests/ha.yaml
+	@echo "---" >> manifests/ha.yaml
+	@cat manifests/ha-webhooks-cc.yaml >> manifests/ha.yaml
 
 # Run go fmt against code
 fmt:
@@ -200,14 +219,25 @@ controller-gen:
 #
 # We only delete the deployment if it exists before applying the manifest, because
 # a) deleting the CRDs will cause all the existing CRs to be wiped away;
-# b) if not deleting the deployment, a new image won't be pulled unless the tag changes.
+# b) if we don't delete the deployment, a new image won't be pulled unless the
+#    tag changes, which it frequently won't since we use the "latest" tag during
+#    development.
 deploy: docker-push kubectl manifests
-	-kubectl -n hnc-system delete deployment hnc-controller-manager
-	kubectl apply -f manifests/${HNC_IMG_NAME}.yaml
+	-kubectl -n hnc-system delete deployment --all
+	kubectl apply -f manifests/default.yaml
 
 deploy-watch:
 	kubectl logs -n hnc-system --follow deployment/hnc-controller-manager manager
 
+deploy-ha: docker-push kubectl manifests
+	-kubectl -n hnc-system delete deployment --all
+	kubectl apply -f manifests/ha.yaml
+
+ha-deploy-watch-ha:
+	kubectl logs -n hnc-system --follow deployment/hnc-controller-manager-ha manager
+
+# No need to delete the HA configuration here - everything "extra" that it
+# installs is in hnc-system, which gets deleted by the default manifest.
 undeploy: manifests
 	@echo "********************************************************************************"
 	@echo "********************************************************************************"
@@ -220,15 +250,16 @@ undeploy: manifests
 	@echo "********************************************************************************"
 	@sleep 5
 	@echo "Deleting all CRDs to ensure all finalizers are removed"
-	-kubectl delete -f manifests/hnc-crds.yaml
+	-kubectl delete -f manifests/crds.yaml
 	@echo "Deleting the rest of HNC"
-	-kubectl delete -f manifests/hnc-manager.yaml
+	-kubectl delete -f manifests/default.yaml
+	@echo Please ignore any \'not found\' errors, these are expected.
 
 # Push the docker image
 docker-push: docker-build
 	@echo "Pushing ${HNC_IMG}"
 ifeq ($(CONFIG),kind)
-	kind load docker-image ${HNC_IMG}
+	kind load docker-image ${HNC_IMG} --name ${KIND}
 else
 	docker push ${HNC_IMG}
 endif
@@ -260,7 +291,7 @@ docker-push-multi: buildx-setup generate fmt vet
 kind-reboot:
 	@echo "Warning: the 'kind' command must be in your path for this to work"
 	-kind delete cluster
-	kind create cluster
+	kind create cluster --name ${KIND}
 
 # Creates a local kind cluster, destroying the old one if necessary. It's not
 # *necessary* to call this wih CONFIG=kind but it's not a bad idea either so
@@ -375,7 +406,7 @@ endif
 	@echo "Starting build."
 	@echo "*********************************************"
 	@echo "*********************************************"
-	gcloud builds submit --config cloudbuild.yaml --no-source --substitutions=${HNC_GCB_SUBS} --timeout=30m
+	gcloud builds submit --config cloudbuild.yaml --no-source --substitutions=${HNC_GCB_SUBS} --timeout=60m
 	@echo "*********************************************"
 	@echo "*********************************************"
 	@echo "Pushing ${HNC_IMG} to ${HNC_RELEASE_IMG}"

cloudbuild.yaml

@@ -27,18 +27,30 @@ steps:
     echo "Building HNC manifests and plugin for $$HNC_REGISTRY/$$HNC_IMG_NAME:$$HNC_IMG_TAG"
     make build
     make krew-build
-# Upload manifest
+# Upload default manifest
 - name: gcr.io/cloud-builders/curl
   args:
   - '-X'
   - 'POST'
   - '-H'
   - 'Content-Type: application/x-application'
   - '--data-binary'
-  - '@hierarchical-namespaces/manifests/hnc-manager.yaml'
+  - '@hierarchical-namespaces/manifests/default.yaml'
   - '-u'
   - '$_HNC_USER:$_HNC_PERSONAL_ACCESS_TOKEN'
-  - 'https://uploads.github.com/repos/$_HNC_REPO_OWNER/hierarchical-namespaces/releases/$_HNC_RELEASE_ID/assets?name=hnc-manager.yaml'
+  - 'https://uploads.github.com/repos/$_HNC_REPO_OWNER/hierarchical-namespaces/releases/$_HNC_RELEASE_ID/assets?name=default.yaml'
+# Upload HA manifest
+- name: gcr.io/cloud-builders/curl
+  args:
+  - '-X'
+  - 'POST'
+  - '-H'
+  - 'Content-Type: application/x-application'
+  - '--data-binary'
+  - '@hierarchical-namespaces/manifests/ha.yaml'
+  - '-u'
+  - '$_HNC_USER:$_HNC_PERSONAL_ACCESS_TOKEN'
+  - 'https://uploads.github.com/repos/$_HNC_REPO_OWNER/hierarchical-namespaces/releases/$_HNC_RELEASE_ID/assets?name=ha-experimental.yaml'
 # Upload plugin (Linux)
 - name: gcr.io/cloud-builders/curl
   args:

config/default/kustomization.yaml

@@ -21,6 +21,8 @@ spec:
     metadata:
       labels:
         control-plane: controller-manager
+      annotations:
+        prometheus.io/scrape: 'true'
     spec:
       securityContext:
         # Generally to run as non-root, the GID and UID can be any number
@@ -32,7 +34,9 @@ spec:
         runAsNonRoot: true
         runAsUser: 1000
       containers:
-      - command:
+      - name: manager
+        image: controller:latest # this is usually overridden by kustomize
+        command:
         - /manager
         args:
         - "--webhook-server-port=9443"
@@ -42,14 +46,21 @@ spec:
         - "--metrics-addr=:8080"
         - "--max-reconciles=10"
         - "--apiserver-qps-throttle=50"
-        - "--enable-internal-cert-management"
-        - "--cert-restart-on-secret-refresh"
         - "--excluded-namespace=kube-system"
         - "--excluded-namespace=kube-public"
         - "--excluded-namespace=hnc-system"
         - "--excluded-namespace=kube-node-lease"
-        image: controller:latest
-        name: manager
+        ports:
+        - containerPort: 9443
+          name: webhook-server
+          protocol: TCP
+        - containerPort: 8080
+          name: metrics
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /tmp/k8s-webhook-server/serving-certs
+          name: cert
+          readOnly: true
         livenessProbe:
           httpGet:
             path: /healthz
@@ -69,4 +80,9 @@ spec:
           requests:
             cpu: 100m
             memory: 150Mi
+      volumes:
+      - name: cert
+        secret:
+          defaultMode: 420
+          secretName: webhook-server-cert
       terminationGracePeriodSeconds: 10

config/default/manager_auth_proxy_patch.yaml

@@ -0,0 +1,4 @@
+This directory is the default configuration of HNC, including internal cert
+management ("cc" = "Cert Controller"). It's published as "default_cc.yaml" when
+building the manifests ("made manifests" in the root directory) and aliased as
+"default.yaml." In versions of HNC prior to v1.0, this was hnc_manager.yaml.

config/default/manager_prometheus_metrics_patch.yaml

@@ -0,0 +1,30 @@
+# Adds namespace to all resources.
+namespace: hnc-system
+
+# Value of this field is prepended to the
+# names of all resources, e.g. a deployment named
+# "wordpress" becomes "alices-wordpress".
+# Note that it should also match with the prefix (text before '-') of the namespace
+# field above.
+namePrefix: hnc-
+
+bases:
+- ../../crd
+- ../../internalcert
+- ../../manager
+- ../../rbac
+- ../../webhook
+
+patches:
+- patch: |-
+    - op: add
+      path: /spec/template/spec/containers/0/args/-
+      value: --enable-internal-cert-management
+    - op: add
+      path: /spec/template/spec/containers/0/args/-
+      value: --cert-restart-on-secret-refresh
+  target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: controller-manager