setup kind with local registry

Signed-off-by: Per Goncalves da Silva <[email protected]>

Author: Per Goncalves da Silva

Makefile

@@ -63,7 +63,7 @@ export CONFIGMAP_SERVER_IMAGE ?= quay.io/operator-framework/configmap-operator-r
 
 PKG := github.com/operator-framework/operator-lifecycle-manager
 IMAGE_REPO ?= quay.io/operator-framework/olm
-IMAGE_TAG ?= "dev"
+IMAGE_TAG ?= "local"
 
 # Go build settings #
 
@@ -102,8 +102,8 @@ KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use -p path $(KUBE_MINOR).x)
 
 # Kind node image tags are in the format x.y.z we pin to version x.y.0 because patch releases and node images
 # are not guaranteed to be available when a new version of the kube apis is released
-KIND_CLUSTER_IMAGE := kindest/node:v$(KUBE_MINOR).0
-KIND_CLUSTER_NAME ?= kind-olmv0
+export KIND_CLUSTER_IMAGE := kindest/node:v$(KUBE_MINOR).0
+export KIND_CLUSTER_NAME ?= kind-olmv0
 
 # Targets #
 # Disable -j flag for make
@@ -145,6 +145,10 @@ image: export GOOS = linux
 image: clean build #HELP Build image image for linux on host architecture
 	docker build -t $(IMAGE_REPO):$(IMAGE_TAG) -f Dockerfile bin
 
+.PHONE: build-e2e-fixture-images
+build-e2e-fixture-images: #HELP Build images for e2e testing
+	./scripts/e2e_test_fixtures.sh
+
 .PHONY: e2e-build
 # the e2e and experimental_metrics tags are required to get e2e tests to pass
 # search the code for go:build e2e or go:build experimental_metrics to see where these tags are used
@@ -207,13 +211,17 @@ kind-clean: $(KIND) #HELP Delete kind cluster $KIND_CLUSTER_NAME (default: kind-
 
 .PHONY: kind-create
 kind-create: kind-clean #HELP Create a new kind cluster $KIND_CLUSTER_NAME (default: kind-olmv0)
-	$(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image $(KIND_CLUSTER_IMAGE) $(KIND_CREATE_OPTS)
+	./scripts/kind_with_registry.sh
 	$(KIND) export kubeconfig --name $(KIND_CLUSTER_NAME)
 
+.PHONY: load-test-images
+load-test-images: #HELP Load the OLM images into the kind cluster's registry
+	./scripts/e2e_test_fixtures.sh --push-to="localhost:5001"
+
 .PHONY: deploy
-OLM_IMAGE := quay.io/operator-framework/olm:local
+OLM_IMAGE ?= $(IMAGE_REPO):$(IMAGE_TAG)
 deploy: $(KIND) $(HELM) #HELP Deploy OLM to kind cluster $KIND_CLUSTER_NAME (default: kind-olmv0) using $OLM_IMAGE (default: quay.io/operator-framework/olm:local)
-	$(KIND) load docker-image $(OLM_IMAGE) --name $(KIND_CLUSTER_NAME); \
+	$(KIND) load docker-image $(OLM_IMAGE) --name $(KIND_CLUSTER_NAME)
 	$(HELM) upgrade --install olm deploy/chart \
 		--set debug=true \
 		--set olm.image.ref=$(OLM_IMAGE) \
@@ -248,7 +256,7 @@ E2E_TEST_NS ?= operators
 E2E_INSTALL_NS ?= operator-lifecycle-manager
 E2E_CATALOG_NS ?= $(E2E_INSTALL_NS)
 E2E_FLAKE_ATTEMPTS ?= 1
-GINKGO_OPTS += -v -randomize-suites -race -trace --show-node-events --flake-attempts=$(E2E_FLAKE_ATTEMPTS) $(if $(E2E_SEED),-seed '$(E2E_SEED)') $(if $(TEST),-focus '$(TEST)',) $(if $(SKIP), -skip '$(SKIP)')
+GINKGO_OPTS += -v -randomize-suites -race -trace --show-node-events -flake-attempts='$(E2E_FLAKE_ATTEMPTS)' $(if $(E2E_SEED),-seed '$(E2E_SEED)' ) $(if $(TEST),-focus '$(TEST)', ) $(if $(SKIP), -skip '$(SKIP)')
 
 .PHONY: e2e
 e2e: #HELP Run e2e tests against a cluster running OLM (params: $E2E_TEST_NS (operator), $E2E_INSTALL_NS (operator-lifecycle-manager), $E2E_CATALOG_NS (operator-lifecycle-manager), $E2E_TIMEOUT (90m), $E2E_FLAKE_ATTEMPTS (1), $TEST(undefined))

scripts/build_test_images.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+KIND=${KIND:-kind}
+CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-docker}
+
+# Default values
+OPERATOR_REGISTRY_VERSION="${OPERATOR_REGISTRY_VERSION:-$(go list -m github.com/operator-framework/operator-registry | cut -d" " -f2 | sed 's/^v//')}"
+KIND_CLUSTER_NAME="${KIND_CLUSTER_NAME:-kind-olmv0}"
+REGISTRY="${REGISTRY:-localhost:5001}"
+
+# Fixtures
+# Note: the following catalogs reference bundles stored in quay.io/olmtest
+INDEX_V1="${REGISTRY}/busybox-dependencies-index:1.0.0-with-ListBundles-method-${OPM_VERSION}"
+INDEX_V2="${REGISTRY}/busybox-dependencies-index:2.0.0-with-ListBundles-method-${OPM_VERSION}"
+TEST_CATALOG_IMAGE="${REGISTRY}/test-catalog:e2e"
+
+## Build
+${CONTAINER_RUNTIME} build -t "${INDEX_V1}" --build-arg="OPM_VERSION=v${OPERATOR_REGISTRY_VERSION}" -f ./test/images/busybox-index/index.Dockerfile ./test/images/busybox-index/indexv1
+${CONTAINER_RUNTIME} build -t "${INDEX_V2}" --build-arg="OPM_VERSION=v${OPERATOR_REGISTRY_VERSION}" -f ./test/images/busybox-index/index.Dockerfile ./test/images/busybox-index/indexv2
+
+# The following catalog used for e2e tests related to serving an extracted registry
+# See catalog_e2e_test.go
+# let's just reuse one of the other catalogs for this - the tests don't care about the content
+# only that a catalog's content can be extracted and served by a different container
+${CONTAINER_RUNTIME} tag "${INDEX_V2}" "${TEST_CATALOG_IMAGE}"
+
+### Push
+${CONTAINER_RUNTIME} push "${INDEX_V1}"
+${CONTAINER_RUNTIME} push "${INDEX_V2}"
+${CONTAINER_RUNTIME} push "${TEST_CATALOG_IMAGE}"

scripts/e2e_test_fixtures.sh

@@ -0,0 +1,77 @@
+#!/bin/sh
+set -o errexit
+set -x
+
+. ".bingo/variables.env"
+
+# Original source: https://kind.sigs.k8s.io/docs/user/local-registry/
+KIND_CLUSTER_NAME=${KIND_CLUSTER_NAME:-kind-olmv0}
+KIND=${KIND:-kind}
+DOCKER=${DOCKER:-docker}
+KUBECTL=${KUBECTL:-kubectl}
+
+# 1. Create registry container unless it already exists
+reg_name='kind-registry'
+reg_port='5001'
+if [ "$(${DOCKER} inspect -f '{{.State.Running}}' "${reg_name}" 2>/dev/null || true)" != 'true' ]; then
+  ${DOCKER} run \
+    -d --restart=always -p "127.0.0.1:${reg_port}:5000" --network bridge --name "${reg_name}" \
+    registry:2
+fi
+
+# 2. Create kind cluster with containerd registry config dir enabled
+# TODO: kind will eventually enable this by default and this patch will
+# be unnecessary.
+#
+# See:
+# https://github.com/kubernetes-sigs/kind/issues/2875
+# https://github.com/containerd/containerd/blob/main/docs/cri/config.md#registry-configuration
+# See: https://github.com/containerd/containerd/blob/main/docs/hosts.md
+if [ -n "${KIND_CLUSTER_IMAGE}" ]; then
+  KIND_IMAGE="--image=${KIND_CLUSTER_IMAGE}"
+fi
+cat <<EOF | ${KIND} create cluster --name="${KIND_CLUSTER_NAME}" "${KIND_IMAGE}" ${KIND_CREATE_OPTS} --config=-
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry]
+    config_path = "/etc/containerd/certs.d"
+EOF
+
+# 3. Add the registry config to the nodes
+#
+# This is necessary because localhost resolves to loopback addresses that are
+# network-namespace local.
+# In other words: localhost in the container is not localhost on the host.
+#
+# We want a consistent name that works from both ends, so we tell containerd to
+# alias localhost:${reg_port} to the registry container when pulling images
+REGISTRY_DIR="/etc/containerd/certs.d/localhost:${reg_port}"
+for node in $(${KIND} --name="${KIND_CLUSTER_NAME}" get nodes); do
+  ${DOCKER} exec "${node}" mkdir -p "${REGISTRY_DIR}"
+  cat <<EOF | ${DOCKER} exec -i "${node}" cp /dev/stdin "${REGISTRY_DIR}/hosts.toml"
+[host."http://${reg_name}:5000"]
+EOF
+done
+
+# 4. Connect the registry to the cluster network
+# This allows kind to bootstrap the network but ensures they're on the same network
+if [ "$(docker inspect -f='{{json .NetworkSettings.Networks.kind}}' "${reg_name}")" = 'null' ]; then
+  docker network connect "kind" "${reg_name}"
+fi
+
+# 5. Document the local registry
+# https://github.com/kubernetes/enhancements/tree/master/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry
+${KIND} export kubeconfig --name="${KIND_CLUSTER_NAME}"
+cat <<EOF | ${KUBECTL} apply -f -
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: local-registry-hosting
+  namespace: kube-public
+data:
+  localRegistryHosting.v1: |
+    host: "localhost:${reg_port}"
+    help: "https://kind.sigs.k8s.io/docs/user/local-registry/"
+EOF