diff --git a/.dockerignore b/.dockerignore index fa9e087b49..d0539097bf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,22 +1,34 @@ .git .github .vscode - -.dockerignore .gitignore .golangci.yml -*.out bin/ -hack/.bin/ -hack/tools/bin/ +**/*.yaml +hack/ out/ docs/ -scripts/ +packaging/ +templates/ **/*.md -*.test -cluster-api-provider-vsphere -examples/provider-components/provider-components*.yaml -test/ +**/.tiltbuild +**/config/**/*.yaml +**/config/**/*.yaml-e +_artifacts +Makefile +**/Makefile + +# ignores changes to test-only code to avoid extra rebuilds +test/e2e/** + +# We want to ignore any frequently modified files to avoid cache-busting the COPY ./ ./ +# Binaries for programs and plugins +**/*.exe +**/*.dll +**/*.so +**/*.dylib +**/bin/** +**/out/** # go.work files go.work diff --git a/Dockerfile b/Dockerfile index 151af9ab7c..4cbcf7d3aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,13 +15,22 @@ # limitations under the License. # Build the manager binary -ARG GOLANG_VERSION=golang:1.20.12 -FROM --platform=${BUILDPLATFORM} ${GOLANG_VERSION} as builder +# Run this with docker build --build-arg builder_image= +ARG builder_image + +# Build architecture +ARG ARCH + +# Ignore Hadolint rule "Always tag the version of an image explicitly." +# It's an invalid finding since the image is explicitly set in the Makefile. +# https://github.com/hadolint/hadolint/wiki/DL3006 +# hadolint ignore=DL3006 +FROM ${builder_image} as builder WORKDIR /workspace -# Run this with docker build --build_arg $(go env GOPROXY) to override the goproxy +# Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy ARG goproxy=https://proxy.golang.org -ENV GOPROXY=${goproxy} +ENV GOPROXY=$goproxy # Copy the Go Modules manifests COPY go.mod go.mod @@ -32,22 +41,30 @@ COPY go.sum go.sum RUN --mount=type=cache,target=/go/pkg/mod \ go mod download +# Copy the sources +COPY ./ ./ + +# Cache the go build into the Go’s compiler cache folder so we take benefits of compiler caching across docker build calls +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + go build . + # Build -ARG TARGETOS -ARG TARGETARCH +ARG package=. +ARG ARCH ARG ldflags -RUN --mount=type=bind,target=. \ - --mount=type=cache,target=/root/.cache/go-build \ + +# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder +RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ - CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -a -ldflags "${ldflags} -extldflags '-static'" \ - -o /out/manager . + CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ + go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \ + -o manager ${package} + -# Copy the controller-manager into a thin image -ARG TARGETPLATFORM -FROM --platform=${TARGETPLATFORM} gcr.io/distroless/static:nonroot +FROM gcr.io/distroless/static:nonroot-${ARCH} WORKDIR / -COPY --from=builder /out/manager . -# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying PSPs +COPY --from=builder /workspace/manager . +# Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies USER 65532 ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index a6995c7f02..60ea159bd5 100644 --- a/Makefile +++ b/Makefile @@ -57,6 +57,7 @@ ROOT_DIR:=$(shell dirname $(realpath $(firstword $(MAKEFILE_LIST)))) BIN_DIR := bin BUILD_DIR := .build TEST_DIR := test +VCSIM_DIR := test/infrastructure/vcsim TOOLS_DIR := hack/tools TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR)) FLAVOR_DIR := $(ROOT_DIR)/templates @@ -242,10 +243,10 @@ MANIFEST_ROOT ?= ./config CRD_ROOT ?= $(MANIFEST_ROOT)/default/crd/bases SUPERVISOR_CRD_ROOT ?= $(MANIFEST_ROOT)/supervisor/crd VMOP_CRD_ROOT ?= $(MANIFEST_ROOT)/deployments/integration-tests/crds -VCSIM_CRD_ROOT ?= test/infrastructure/vcsim/config/crd/bases +VCSIM_CRD_ROOT ?= $(VCSIM_DIR)/config/crd/bases WEBHOOK_ROOT ?= $(MANIFEST_ROOT)/webhook RBAC_ROOT ?= $(MANIFEST_ROOT)/rbac -VCSIM_RBAC_ROOT ?= test/infrastructure/vcsim/config/rbac +VCSIM_RBAC_ROOT ?= $(VCSIM_DIR)/config/rbac VERSION ?= $(shell cat clusterctl-settings.json | jq .config.nextVersion -r) OVERRIDES_DIR := $(HOME)/.cluster-api/overrides/infrastructure-vsphere/$(VERSION) @@ -288,11 +289,11 @@ generate-manifests: $(CONTROLLER_GEN) ## Generate manifests e.g. CRD, RBAC etc. output:crd:dir=$(VMOP_CRD_ROOT) # vcsim crds are used for tests. $(CONTROLLER_GEN) \ - paths=./test/infrastructure/vcsim/api/v1alpha1 \ + paths=./$(VCSIM_DIR)/api/v1alpha1 \ crd:crdVersions=v1 \ output:crd:dir=$(VCSIM_CRD_ROOT) $(CONTROLLER_GEN) \ - paths=./test/infrastructure/vcsim/controllers/... \ + paths=./$(VCSIM_DIR)/controllers/... \ output:rbac:dir=$(VCSIM_RBAC_ROOT) \ rbac:roleName=manager-role @@ -304,7 +305,7 @@ generate-go-deepcopy: $(CONTROLLER_GEN) ## Generate deepcopy go code for core paths=./apis/... $(CONTROLLER_GEN) \ object:headerFile=./hack/boilerplate/boilerplate.generatego.txt \ - paths=./test/infrastructure/vcsim/api/... + paths=./$(VCSIM_DIR)/api/... .PHONY: generate-modules generate-modules: ## Run go mod tidy to ensure modules are up to date @@ -488,7 +489,8 @@ DOCKER_BUILD_MODIFY_MANIFESTS ?= true .PHONY: docker-build docker-build: docker-pull-prerequisites ## Build the docker image for vsphere controller manager - DOCKER_BUILDKIT=1 docker build --platform linux/$(ARCH) --build-arg GOLANG_VERSION=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) +## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368) + cat ./Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) --file - @if [ "${DOCKER_BUILD_MODIFY_MANIFESTS}" = "true" ]; then \ $(MAKE) set-manifest-image MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./config/base/manager_image_patch.yaml"; \ $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./config/base/manager_pull_policy.yaml"; \ @@ -496,10 +498,11 @@ docker-build: docker-pull-prerequisites ## Build the docker image for vsphere co .PHONY: docker-build-vcsim docker-build-vcsim: docker-pull-prerequisites ## Build the docker image for vcsim controller manager - DOCKER_BUILDKIT=1 docker build --platform linux/$(ARCH) --build-arg GOLANG_VERSION=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ldflags="$(LDFLAGS)" . -t $(VCSIM_CONTROLLER_IMG)-$(ARCH):$(TAG) +## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368) + cat $(VCSIM_DIR)/Dockerfile | DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(VCSIM_CONTROLLER_IMG)-$(ARCH):$(TAG) --file - @if [ "${DOCKER_BUILD_MODIFY_MANIFESTS}" = "true" ]; then \ - $(MAKE) set-manifest-image MANIFEST_IMG=$(VCSIM_CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./test/infrastructure/vcsim/config/default/manager_image_patch.yaml"; \ - $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./test/infrastructure/vcsim/config/default/manager_pull_policy.yaml"; \ + $(MAKE) set-manifest-image MANIFEST_IMG=$(VCSIM_CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./$(VCSIM_DIR)/config/default/manager_image_patch.yaml"; \ + $(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./$(VCSIM_DIR)/config/default/manager_pull_policy.yaml"; \ fi ## -------------------------------------- @@ -539,17 +542,19 @@ test-cover: ## Run unit tests and generate a coverage report go tool cover -html=coverage.out -o coverage.html .PHONY: test-integration -test-integration: e2e-image ## Run integration tests +test-integration: e2e-images ## Run integration tests test-integration: $(GINKGO) $(KUSTOMIZE) $(KIND) time $(GINKGO) --output-dir="$(ARTIFACTS)" --junit-report="junit.integration_suite.1.xml" -v ./test/integration -- --config=$(INTEGRATION_CONF_FILE) --artifacts-folder="$(ARTIFACTS)" -.PHONY: e2e-image -e2e-image: ## Build the e2e manager image - docker buildx build --platform linux/$(ARCH) --output=type=docker \ - --build-arg ldflags="$(LDFLAGS)" --tag="gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:dev" . +.PHONY: e2e-images +e2e-images: ## Build the e2e manager image + # please ensure the generated image name matches image names used in the E2E_CONF_FILE; + # also the same settings must exist in e2e.sh + $(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build + $(MAKE) REGISTRY=gcr.io/k8s-staging-capi-vsphere PULL_POLICY=IfNotPresent TAG=dev docker-build-vcsim .PHONY: e2e -e2e: e2e-image generate-e2e-templates +e2e: e2e-images generate-e2e-templates e2e: $(GINKGO) $(KUSTOMIZE) $(KIND) $(GOVC) ## Run e2e tests @echo PATH="$(PATH)" @echo diff --git a/hack/e2e.sh b/hack/e2e.sh index 408bd61c61..d5e5215db4 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -94,7 +94,7 @@ make envsubst # Only build and upload the image if we run tests which require it to save some $. if [[ -z "${GINKGO_FOCUS+x}" ]]; then # Save the docker image locally - make e2e-image + make e2e-images mkdir -p /tmp/images docker save gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:dev -o "$DOCKER_IMAGE_TAR" diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index a8b239f2e5..a4aa6ff06c 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -14,7 +14,7 @@ images: loadBehavior: tryLoad - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.6.1 loadBehavior: tryLoad - - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:dev + - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-{ARCH}:dev loadBehavior: mustLoad - name: quay.io/jetstack/cert-manager-cainjector:v1.12.2 loadBehavior: tryLoad @@ -104,11 +104,6 @@ providers: # Use manifest from source files value: ../../../../cluster-api-provider-vsphere/config/default contract: v1beta1 - replacements: - - old: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:main - new: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:dev - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" files: # Add a cluster template - sourcePath: "../../../test/e2e/data/infrastructure-vsphere/main/cluster-template-conformance.yaml" diff --git a/test/framework/vmoperator/vmoperator.go b/test/framework/vmoperator/vmoperator.go index 80933c407b..e10b4c650c 100644 --- a/test/framework/vmoperator/vmoperator.go +++ b/test/framework/vmoperator/vmoperator.go @@ -41,7 +41,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" "sigs.k8s.io/cluster-api-provider-vsphere/pkg/session" ) @@ -342,7 +341,7 @@ func ReconcileDependencies(ctx context.Context, c client.Client, config Dependen }, ClassRef: vmoprv1.ClassReference{ APIVersion: vmoprv1.SchemeGroupVersion.String(), - Kind: util.TypeToKind(&vmoprv1.VirtualMachineClass{}), + Kind: "VirtualMachineClass", Name: vmClass.Name, }, } @@ -430,7 +429,7 @@ func ReconcileDependencies(ctx context.Context, c client.Client, config Dependen }, ContentSourceRef: vmoprv1.ContentSourceReference{ APIVersion: vmoprv1.SchemeGroupVersion.String(), - Kind: util.TypeToKind(&vmoprv1.ContentSource{}), + Kind: "ContentSource", Name: contentSource.Name, }, } diff --git a/test/infrastructure/vcsim/Dockerfile b/test/infrastructure/vcsim/Dockerfile index ed964e1e97..c4cd873767 100644 --- a/test/infrastructure/vcsim/Dockerfile +++ b/test/infrastructure/vcsim/Dockerfile @@ -49,7 +49,7 @@ RUN --mount=type=cache,target=/go/pkg/mod \ # This needs to build with the entire Cluster API context WORKDIR /workspace -# Copy the sources (which includes the test/infrastructure/inmemory subdirectory) +# Copy the sources (which includes the test/infrastructure/vcsim subdirectory) COPY ./ ./ # Essentially, change directories into vcsim @@ -61,6 +61,7 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ go build . # Build +ARG package=. ARG ARCH ARG ldflags @@ -69,12 +70,12 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ --mount=type=cache,target=/go/pkg/mod \ CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \ - -o /workspace/manager . + -o manager ${package} FROM gcr.io/distroless/static:nonroot-${ARCH} WORKDIR / -COPY --from=builder /workspace/manager . +COPY --from=builder /workspace/test/infrastructure/vcsim/manager . # Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies USER 65532 ENTRYPOINT ["/manager"] diff --git a/test/integration/integration-dev.yaml b/test/integration/integration-dev.yaml index ddf48b9eec..dd61947a86 100644 --- a/test/integration/integration-dev.yaml +++ b/test/integration/integration-dev.yaml @@ -17,7 +17,7 @@ images: loadBehavior: tryLoad - name: registry.k8s.io/cluster-api/kubeadm-control-plane-controller:v1.6.1 loadBehavior: tryLoad - - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:dev + - name: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller-{ARCH}:dev loadBehavior: mustLoad - name: quay.io/jetstack/cert-manager-cainjector:v1.12.2 loadBehavior: tryLoad @@ -78,11 +78,7 @@ providers: contract: v1beta1 files: - sourcePath: "../e2e/data/shared/main/v1beta1_provider/metadata.yaml" - replacements: - - old: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:main - new: gcr.io/k8s-staging-capi-vsphere/cluster-api-vsphere-controller:dev - - old: "imagePullPolicy: Always" - new: "imagePullPolicy: IfNotPresent" + variables: VSPHERE_PASSWORD: "admin123" VSPHERE_USERNAME: "admin123"