Skip to content

Commit abd279e

Browse files
mayuka-cmayuka-c
committed
[e2e] Fixing flakes in multigroup serve metrics
Fix flakes - 2 Run make generate RUn make generate Minor fix Minor fix Fix-2 Fix-3 Implement a new marker for webhook readiness Lint fix scaffold md fix Address comment-2 Minor fix Test-1
1 parent 17fc377 commit abd279e

File tree

8 files changed

+235
-17
lines changed

8 files changed

+235
-17
lines changed

‎docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,39 @@ var _ = Describe("Manager", Ordered, func() {
216216
}
217217
Eventually(verifyMetricsServerStarted).Should(Succeed())
218218

219+
By("waiting for webhook service to be ready")
220+
verifyWebhookServiceReady := func(g Gomega) {
221+
const webhookServiceName = "project-webhook-service"
222+
223+
// Webhook service should exist since webhooks are configured
224+
cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
225+
_, err := utils.Run(cmd)
226+
g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")
227+
228+
// Check if webhook server is ready by verifying pod readiness
229+
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
230+
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
231+
output, err := utils.Run(cmd)
232+
g.Expect(err).NotTo(HaveOccurred())
233+
g.Expect(output).To(Equal("True"),
234+
"Controller manager pod not ready (webhook server may not be accepting connections)")
235+
236+
// Check if webhook service endpoints are available
237+
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
238+
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
239+
output, err = utils.Run(cmd)
240+
g.Expect(err).NotTo(HaveOccurred())
241+
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")
242+
243+
// Test webhook connectivity by checking if webhook server port is responding
244+
cmd = exec.Command("kubectl", "get", "--raw",
245+
fmt.Sprintf("/api/v1/namespaces/%s/services/%s/proxy/readyz", namespace, webhookServiceName))
246+
_, err = utils.Run(cmd)
247+
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
248+
}
249+
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
250+
// +kubebuilder:scaffold:e2e-webhooks-readiness
251+
219252
By("creating the curl-metrics pod to access the metrics endpoint")
220253
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
221254
"--namespace", namespace,

‎docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,8 @@ var _ = Describe("Manager", Ordered, func() {
211211
}
212212
Eventually(verifyMetricsServerStarted).Should(Succeed())
213213

214+
// +kubebuilder:scaffold:e2e-webhooks-readiness
215+
214216
By("creating the curl-metrics pod to access the metrics endpoint")
215217
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
216218
"--namespace", namespace,

‎docs/book/src/multiversion-tutorial/testdata/project/test/e2e/e2e_test.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,39 @@ var _ = Describe("Manager", Ordered, func() {
216216
}
217217
Eventually(verifyMetricsServerStarted).Should(Succeed())
218218

219+
By("waiting for webhook service to be ready")
220+
verifyWebhookServiceReady := func(g Gomega) {
221+
const webhookServiceName = "project-webhook-service"
222+
223+
// Webhook service should exist since webhooks are configured
224+
cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
225+
_, err := utils.Run(cmd)
226+
g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")
227+
228+
// Check if webhook server is ready by verifying pod readiness
229+
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
230+
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
231+
output, err := utils.Run(cmd)
232+
g.Expect(err).NotTo(HaveOccurred())
233+
g.Expect(output).To(Equal("True"),
234+
"Controller manager pod not ready (webhook server may not be accepting connections)")
235+
236+
// Check if webhook service endpoints are available
237+
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
238+
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
239+
output, err = utils.Run(cmd)
240+
g.Expect(err).NotTo(HaveOccurred())
241+
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")
242+
243+
// Test webhook connectivity by checking if webhook server port is responding
244+
cmd = exec.Command("kubectl", "get", "--raw",
245+
fmt.Sprintf("/api/v1/namespaces/%s/services/%s/proxy/readyz", namespace, webhookServiceName))
246+
_, err = utils.Run(cmd)
247+
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
248+
}
249+
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
250+
// +kubebuilder:scaffold:e2e-webhooks-readiness
251+
219252
By("creating the curl-metrics pod to access the metrics endpoint")
220253
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
221254
"--namespace", namespace,

‎docs/book/src/reference/markers/scaffold.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ properly registered with the manager, so that the controller can reconcile the r
107107
| `+kubebuilder:scaffold:crdkustomizecainjectioname` | `config/default` | Marks where CA injection patches are added for the conversion webhooks. |
108108
| **(No longer supported)** `+kubebuilder:scaffold:crdkustomizecainjectionpatch` | `config/crd` | Marks where CA injection patches are added for the webhooks. Replaced by `+kubebuilder:scaffold:crdkustomizecainjectionns` and `+kubebuilder:scaffold:crdkustomizecainjectioname` |
109109
| `+kubebuilder:scaffold:manifestskustomizesamples` | `config/samples` | Marks where Kustomize sample manifests are injected. |
110+
| `+kubebuilder:scaffold:e2e-webhooks-readiness` | `test/e2e` | Adds webhooks readiness checks in e2e tests before metrics collection. |
110111
| `+kubebuilder:scaffold:e2e-webhooks-checks` | `test/e2e` | Adds e2e checks for webhooks depending on the types of webhooks scaffolded. |
111112

112113
<aside class="note warning">

‎pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go‎

Lines changed: 67 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
log "log/slog"
2323
"os"
2424
"path/filepath"
25+
"strings"
2526

2627
"sigs.k8s.io/kubebuilder/v4/pkg/machinery"
2728
)
@@ -31,7 +32,10 @@ var (
3132
_ machinery.Inserter = &WebhookTestUpdater{}
3233
)
3334

34-
const webhookChecksMarker = "e2e-webhooks-checks"
35+
const (
36+
webhookChecksMarker = "e2e-webhooks-checks"
37+
webhookReadinessMarker = "e2e-webhooks-readiness"
38+
)
3539

3640
// Test defines the basic setup for the e2e test
3741
type Test struct {
@@ -74,6 +78,7 @@ func (*WebhookTestUpdater) GetIfExistsAction() machinery.IfExistsAction {
7478
// GetMarkers implements file.Inserter
7579
func (f *WebhookTestUpdater) GetMarkers() []machinery.Marker {
7680
return []machinery.Marker{
81+
machinery.NewMarkerFor(f.GetPath(), webhookReadinessMarker),
7782
machinery.NewMarkerFor(f.GetPath(), webhookChecksMarker),
7883
}
7984
}
@@ -107,25 +112,35 @@ func (f *WebhookTestUpdater) GetCodeFragments() machinery.CodeFragmentsMap {
107112
}
108113

109114
var fragments []string
110-
fragments = append(fragments, webhookChecksFragment)
111115

112-
if f.Resource != nil && f.Resource.HasDefaultingWebhook() {
113-
mutatingWebhookCode := fmt.Sprintf(mutatingWebhookChecksFragment, f.ProjectName)
114-
fragments = append(fragments, mutatingWebhookCode)
115-
}
116+
// Handle different markers differently
117+
markerString := marker.String()
118+
if strings.Contains(markerString, webhookReadinessMarker) {
119+
// Only inject readiness check for this marker
120+
webhookReadinessCode := fmt.Sprintf(webhookReadinessFragment, f.ProjectName)
121+
fragments = append(fragments, webhookReadinessCode)
122+
} else if strings.Contains(markerString, webhookChecksMarker) {
123+
// Inject all other webhook tests for this marker
124+
fragments = append(fragments, webhookChecksFragment)
125+
126+
if f.Resource != nil && f.Resource.HasDefaultingWebhook() {
127+
mutatingWebhookCode := fmt.Sprintf(mutatingWebhookChecksFragment, f.ProjectName)
128+
fragments = append(fragments, mutatingWebhookCode)
129+
}
116130

117-
if f.Resource != nil && f.Resource.HasValidationWebhook() {
118-
validatingWebhookCode := fmt.Sprintf(validatingWebhookChecksFragment, f.ProjectName)
119-
fragments = append(fragments, validatingWebhookCode)
120-
}
131+
if f.Resource != nil && f.Resource.HasValidationWebhook() {
132+
validatingWebhookCode := fmt.Sprintf(validatingWebhookChecksFragment, f.ProjectName)
133+
fragments = append(fragments, validatingWebhookCode)
134+
}
121135

122-
if f.Resource != nil && f.Resource.HasConversionWebhook() {
123-
conversionWebhookCode := fmt.Sprintf(
124-
conversionWebhookChecksFragment,
125-
f.Resource.Kind,
126-
f.Resource.Plural+"."+f.Resource.Group+"."+f.Resource.Domain,
127-
)
128-
fragments = append(fragments, conversionWebhookCode)
136+
if f.Resource != nil && f.Resource.HasConversionWebhook() {
137+
conversionWebhookCode := fmt.Sprintf(
138+
conversionWebhookChecksFragment,
139+
f.Resource.Kind,
140+
f.Resource.Plural+"."+f.Resource.Group+"."+f.Resource.Domain,
141+
)
142+
fragments = append(fragments, conversionWebhookCode)
143+
}
129144
}
130145

131146
codeFragments[marker] = fragments
@@ -198,6 +213,39 @@ const conversionWebhookChecksFragment = `It("should have CA injection for %[1]s
198213
199214
`
200215

216+
const webhookReadinessFragment = `By("waiting for webhook service to be ready")
217+
verifyWebhookServiceReady := func(g Gomega) {
218+
const webhookServiceName = "%s-webhook-service"
219+
220+
// Webhook service should exist since webhooks are configured
221+
cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
222+
_, err := utils.Run(cmd)
223+
g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")
224+
225+
// Check if webhook server is ready by verifying pod readiness
226+
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
227+
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
228+
output, err := utils.Run(cmd)
229+
g.Expect(err).NotTo(HaveOccurred())
230+
g.Expect(output).To(Equal("True"),
231+
"Controller manager pod not ready (webhook server may not be accepting connections)")
232+
233+
// Check if webhook service endpoints are available
234+
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
235+
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
236+
output, err = utils.Run(cmd)
237+
g.Expect(err).NotTo(HaveOccurred())
238+
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")
239+
240+
// Test webhook connectivity by checking if webhook server port is responding
241+
cmd = exec.Command("kubectl", "get", "--raw",
242+
fmt.Sprintf("/api/v1/namespaces/%%s/services/%%s/proxy/readyz", namespace, webhookServiceName))
243+
_, err = utils.Run(cmd)
244+
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
245+
}
246+
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
247+
`
248+
201249
var testCodeTemplate = `//go:build e2e
202250
// +build e2e
203251
@@ -394,6 +442,8 @@ var _ = Describe("Manager", Ordered, func() {
394442
}
395443
Eventually(verifyMetricsServerStarted).Should(Succeed())
396444
445+
// +kubebuilder:scaffold:e2e-webhooks-readiness
446+
397447
By("creating the curl-metrics pod to access the metrics endpoint")
398448
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
399449
"--namespace", namespace,

‎testdata/project-v4-multigroup/test/e2e/e2e_test.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,39 @@ var _ = Describe("Manager", Ordered, func() {
211211
}
212212
Eventually(verifyMetricsServerStarted).Should(Succeed())
213213

214+
By("waiting for webhook service to be ready")
215+
verifyWebhookServiceReady := func(g Gomega) {
216+
const webhookServiceName = "project-v4-multigroup-webhook-service"
217+
218+
// Webhook service should exist since webhooks are configured
219+
cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
220+
_, err := utils.Run(cmd)
221+
g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")
222+
223+
// Check if webhook server is ready by verifying pod readiness
224+
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
225+
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
226+
output, err := utils.Run(cmd)
227+
g.Expect(err).NotTo(HaveOccurred())
228+
g.Expect(output).To(Equal("True"),
229+
"Controller manager pod not ready (webhook server may not be accepting connections)")
230+
231+
// Check if webhook service endpoints are available
232+
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
233+
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
234+
output, err = utils.Run(cmd)
235+
g.Expect(err).NotTo(HaveOccurred())
236+
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")
237+
238+
// Test webhook connectivity by checking if webhook server port is responding
239+
cmd = exec.Command("kubectl", "get", "--raw",
240+
fmt.Sprintf("/api/v1/namespaces/%s/services/%s/proxy/readyz", namespace, webhookServiceName))
241+
_, err = utils.Run(cmd)
242+
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
243+
}
244+
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
245+
// +kubebuilder:scaffold:e2e-webhooks-readiness
246+
214247
By("creating the curl-metrics pod to access the metrics endpoint")
215248
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
216249
"--namespace", namespace,

‎testdata/project-v4-with-plugins/test/e2e/e2e_test.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,39 @@ var _ = Describe("Manager", Ordered, func() {
211211
}
212212
Eventually(verifyMetricsServerStarted).Should(Succeed())
213213

214+
By("waiting for webhook service to be ready")
215+
verifyWebhookServiceReady := func(g Gomega) {
216+
const webhookServiceName = "project-v4-with-plugins-webhook-service"
217+
218+
// Webhook service should exist since webhooks are configured
219+
cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
220+
_, err := utils.Run(cmd)
221+
g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")
222+
223+
// Check if webhook server is ready by verifying pod readiness
224+
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
225+
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
226+
output, err := utils.Run(cmd)
227+
g.Expect(err).NotTo(HaveOccurred())
228+
g.Expect(output).To(Equal("True"),
229+
"Controller manager pod not ready (webhook server may not be accepting connections)")
230+
231+
// Check if webhook service endpoints are available
232+
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
233+
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
234+
output, err = utils.Run(cmd)
235+
g.Expect(err).NotTo(HaveOccurred())
236+
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")
237+
238+
// Test webhook connectivity by checking if webhook server port is responding
239+
cmd = exec.Command("kubectl", "get", "--raw",
240+
fmt.Sprintf("/api/v1/namespaces/%s/services/%s/proxy/readyz", namespace, webhookServiceName))
241+
_, err = utils.Run(cmd)
242+
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
243+
}
244+
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
245+
// +kubebuilder:scaffold:e2e-webhooks-readiness
246+
214247
By("creating the curl-metrics pod to access the metrics endpoint")
215248
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
216249
"--namespace", namespace,

‎testdata/project-v4/test/e2e/e2e_test.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,39 @@ var _ = Describe("Manager", Ordered, func() {
211211
}
212212
Eventually(verifyMetricsServerStarted).Should(Succeed())
213213

214+
By("waiting for webhook service to be ready")
215+
verifyWebhookServiceReady := func(g Gomega) {
216+
const webhookServiceName = "project-v4-webhook-service"
217+
218+
// Webhook service should exist since webhooks are configured
219+
cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
220+
_, err := utils.Run(cmd)
221+
g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")
222+
223+
// Check if webhook server is ready by verifying pod readiness
224+
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
225+
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
226+
output, err := utils.Run(cmd)
227+
g.Expect(err).NotTo(HaveOccurred())
228+
g.Expect(output).To(Equal("True"),
229+
"Controller manager pod not ready (webhook server may not be accepting connections)")
230+
231+
// Check if webhook service endpoints are available
232+
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
233+
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
234+
output, err = utils.Run(cmd)
235+
g.Expect(err).NotTo(HaveOccurred())
236+
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")
237+
238+
// Test webhook connectivity by checking if webhook server port is responding
239+
cmd = exec.Command("kubectl", "get", "--raw",
240+
fmt.Sprintf("/api/v1/namespaces/%s/services/%s/proxy/readyz", namespace, webhookServiceName))
241+
_, err = utils.Run(cmd)
242+
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
243+
}
244+
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
245+
// +kubebuilder:scaffold:e2e-webhooks-readiness
246+
214247
By("creating the curl-metrics pod to access the metrics endpoint")
215248
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
216249
"--namespace", namespace,

0 commit comments

Comments
 (0)