Skip to content

Commit 7045c13

Browse files
NishanthNalluriNishanthNalluri
committed
Nightly Integration tests for self-hosted bare metal infrastructure.
1 parent b60abd0 commit 7045c13

File tree

14 files changed

+554
-446
lines changed

14 files changed

+554
-446
lines changed

‎.github/workflows/integration-tests.yaml‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ on:
99
jobs:
1010
integration-tests-multi-region:
1111
runs-on: ubuntu-latest-4-core
12+
strategy:
13+
fail-fast: false
14+
matrix:
15+
provider: [k3d, gcp]
1216
timeout-minutes: 90
1317
permissions:
1418
contents: 'read'
@@ -35,11 +39,11 @@ jobs:
3539
gcloud components install gke-gcloud-auth-plugin
3640
- name: Run tests (multi-region)
3741
env:
38-
isNightly: true
42+
PROVIDER: ${{ matrix.provider }}
3943
USE_GKE_GCLOUD_AUTH_PLUGIN: True
4044
run: |
41-
set -e
42-
make test/nightly-e2e/multi-region | tee test_output.log || true
45+
set -euo pipefail
46+
make test/e2e/multi-region | tee test_output.log
4347
- name: Archive test results
4448
if: ${{ always() }}
4549
uses: actions/upload-artifact@v4
@@ -123,6 +127,10 @@ jobs:
123127
"${{ secrets.SLACK_WEBHOOK_URL }}"
124128
integration-tests-single-region:
125129
runs-on: ubuntu-latest-4-core
130+
strategy:
131+
fail-fast: false
132+
matrix:
133+
provider: [k3d, gcp]
126134
timeout-minutes: 90
127135
permissions:
128136
contents: read
@@ -150,11 +158,11 @@ jobs:
150158
gcloud components install gke-gcloud-auth-plugin
151159
- name: Run tests (single-region)
152160
env:
153-
isNightly: true
161+
PROVIDER: ${{ matrix.provider }}
154162
USE_GKE_GCLOUD_AUTH_PLUGIN: True
155163
run: |
156-
set -e
157-
make test/nightly-e2e/single-region | tee test_output.log || true
164+
set -euo pipefail
165+
make test/e2e/single-region | tee test_output.log
158166
- name: Archive test results
159167
if: ${{ always() }}
160168
uses: actions/upload-artifact@v4

‎Makefile‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ ifeq ($(UNAME_S),Linux)
44
COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.linux-amd64.tgz
55
HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-linux-amd64.tar.gz
66
K3D_BIN ?= https://github.com/k3d-io/k3d/releases/download/v5.7.4/k3d-linux-amd64
7+
KIND_BIN ?= https://kind.sigs.k8s.io/dl/v0.29.0/kind-linux-amd64
78
KUBECTL_BIN ?= https://dl.k8s.io/release/v1.29.1/bin/linux/amd64/kubectl
89
YQ_BIN ?= https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_linux_amd64
910
JQ_BIN ?= https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64
@@ -14,6 +15,7 @@ ifeq ($(UNAME_S),Darwin)
1415
COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.darwin-10.9-amd64.tgz
1516
HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-darwin-amd64.tar.gz
1617
K3D_BIN ?= https://github.com/k3d-io/k3d/releases/download/v5.7.4/k3d-darwin-arm64
18+
KIND_BIN ?= https://kind.sigs.k8s.io/dl/v0.29.0/kind-darwin-arm64
1719
KUBECTL_BIN ?= https://dl.k8s.io/release/v1.29.1/bin/darwin/amd64/kubectl
1820
YQ_BIN ?= https://github.com/mikefarah/yq/releases/download/v4.31.2/yq_darwin_amd64
1921
JQ_BIN ?= https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64
@@ -118,10 +120,10 @@ test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/cluster/up
118120
$(MAKE) test/cluster/down; \
119121
exit $${EXIT_CODE:-0}
120122

121-
test/e2e/multi-region: bin/cockroach bin/kubectl bin/helm build/self-signer
123+
test/e2e/multi-region: bin/cockroach bin/kubectl bin/helm build/self-signer bin/k3d
122124
@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInMultiRegion ./tests/e2e/operator/multiRegion/... || (echo "Multi region tests failed with exit code $$?" && exit 1)
123125

124-
test/e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer
126+
test/e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer bin/k3d
125127
@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || (echo "Single region tests failed with exit code $$?" && exit 1)
126128

127129
test/e2e/migrate: bin/cockroach bin/kubectl bin/helm bin/migration-helper build/self-signer test/cluster/up/3
@@ -135,13 +137,6 @@ test/single-cluster/up: bin/k3d
135137
test/multi-cluster/down: bin/k3d
136138
./tests/k3d/dev-multi-cluster.sh down
137139

138-
test/nightly-e2e/single-region: bin/cockroach bin/kubectl bin/helm build/self-signer
139-
@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInSingleRegion ./tests/e2e/operator/singleRegion/... || (echo "Single region tests failed with exit code $$?" && exit 1)
140-
141-
test/nightly-e2e/multi-region: bin/cockroach bin/kubectl bin/helm build/self-signer
142-
@PATH="$(PWD)/bin:${PATH}" go test -timeout 60m -v -test.run TestOperatorInMultiRegion ./tests/e2e/operator/multiRegion/... || (echo "Multi region tests failed with exit code $$?" && exit 1)
143-
144-
145140
test/lint: bin/helm ## lint the helm chart
146141
@build/lint.sh && \
147142
bin/helm lint cockroachdb && \
@@ -176,6 +171,11 @@ bin/k3d: ## install k3d
176171
@curl -Lo bin/k3d $(K3D_BIN)
177172
@chmod +x bin/k3d
178173

174+
bin/kind: ## install kind
175+
@mkdir -p bin
176+
@curl -Lo bin/kind $(KIND_BIN)
177+
@chmod +x bin/kind
178+
179179
bin/kubectl: ## install kubectl
180180
@mkdir -p bin
181181
@curl -Lo bin/kubectl $(KUBECTL_BIN)

‎cockroachdb-parent/charts/operator/templates/operator.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,7 @@ rules:
254254
- admissionregistration.k8s.io
255255
resources:
256256
- validatingwebhookconfigurations
257+
- mutatingwebhookconfigurations
257258
verbs:
258259
- create
259260
- apiGroups:
@@ -265,6 +266,15 @@ rules:
265266
verbs:
266267
- get
267268
- patch
269+
- apiGroups:
270+
- admissionregistration.k8s.io
271+
resources:
272+
- mutatingwebhookconfigurations
273+
resourceNames:
274+
- cockroach-mutating-webhook-config
275+
verbs:
276+
- get
277+
- patch
268278
# The "create" verb cannot be qualified with resourceNames, so grant the
269279
# unqualified permission so that the operator can create new CRDs. But only
270280
# allow the operator to get and patch its own CRDs.

‎cockroachdb-parent/charts/operator/values.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,13 @@
44
# image captures the container image settings for Operator pods.
55
image:
66
# registry is the container registry where the image is stored.
7-
registry: "us-docker.pkg.dev/releases-prod/self-hosted"
7+
registry: "us-docker.pkg.dev/cockroach-cloud-images/development"
88
# repository defines the image repository.
9-
repository: "cockroachdb-operator@sha256"
9+
repository: "cockroach-operator@sha256"
1010
# pullPolicy specifies the image pull policy.
1111
pullPolicy: IfNotPresent
1212
# tag is the image tag.
13-
tag: "6f62639e9fee99d99b0387a9dccda84daa1a489b592b008f2f354ec57eae09ac"
13+
tag: "72844b85354fd55b9a487abbd6e253b7d5081f65513c3813fde0ceb7d3ee2f70"
1414
# certificate defines the certificate settings for the Operator.
1515
certificate:
1616
# validForDays specifies the number of days the certificate is valid for.

‎tests/e2e/migrate/helm_chart_to_cockroach_enterprise_operator_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ func (h *HelmChartToOperator) TestDefaultMigration(t *testing.T) {
112112
k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
113113
}()
114114

115-
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
115+
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
116116
defer func() {
117117
t.Log("Uninstall the cockroachdb enterprise operator")
118118
operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)
@@ -221,7 +221,7 @@ func (h *HelmChartToOperator) TestCertManagerMigration(t *testing.T) {
221221
k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
222222
}()
223223

224-
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
224+
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
225225
defer func() {
226226
t.Log("Uninstall the cockroachdb enterprise operator")
227227
operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)
@@ -308,7 +308,7 @@ func (h *HelmChartToOperator) TestPCRPrimaryMigration(t *testing.T) {
308308
k8s.RunKubectl(t, kubectlOptions, "delete", "priorityclass", "crdb-critical")
309309
}()
310310

311-
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
311+
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
312312
defer func() {
313313
t.Log("Uninstall the cockroachdb enterprise operator")
314314
operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)

‎tests/e2e/migrate/public_operator_to_cockroach_enterprise_operator_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ func (o *PublicOperatorToCockroachEnterpriseOperator) TestDefaultMigration(t *te
127127
k8s.KubectlApply(t, kubectlOptions, filepath.Join(manifestsDirPath, "rbac.yaml"))
128128

129129
t.Log("Install the cockroachdb enterprise operator")
130-
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions)
130+
operator.InstallCockroachDBEnterpriseOperator(t, kubectlOptions, nil)
131131
defer func() {
132132
t.Log("Uninstall the cockroachdb enterprise operator")
133133
operator.UninstallCockroachDBEnterpriseOperator(t, kubectlOptions)

‎tests/e2e/operator/infra/common.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,11 @@ const (
2424

2525
// Common constants.
2626
const (
27-
defaultRetries = 30
28-
defaultRetryInterval = 10 * time.Second
27+
defaultRetries = 30
28+
defaultRetryInterval = 10 * time.Second
29+
// Load balancer specific retry settings (extended for AWS)
30+
loadBalancerRetries = 60 // 10 minutes total
31+
loadBalancerInterval = 10 * time.Second
2932
coreDNSDeploymentName = "coredns"
3033
coreDNSServiceName = "crl-core-dns"
3134
coreDNSNamespace = "kube-system"
@@ -237,7 +240,7 @@ func finalizeCoreDNSDeployment(t *testing.T, kubectlOpts *k8s.KubectlOptions) er
237240
func WaitForCoreDNSServiceIPs(t *testing.T, kubectlOpts *k8s.KubectlOptions) ([]string, error) {
238241
var ips []string
239242

240-
_, err := retry.DoWithRetryE(t, "waiting for CoreDNS service IPs", defaultRetries, defaultRetryInterval,
243+
_, err := retry.DoWithRetryE(t, "waiting for CoreDNS service IPs", loadBalancerRetries, loadBalancerInterval,
241244
func() (string, error) {
242245
svc, err := k8s.GetServiceE(t, kubectlOpts, coreDNSServiceName)
243246
if err != nil {

0 commit comments

Comments
 (0)