Skip to content

🐛 Fix flakes in e2e tests for metrics #5138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
+235 −17
Open

🐛 Fix flakes in e2e tests for metrics #5138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 33 additions & 0 deletions docs/book/src/cronjob-tutorial/testdata/project/test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,39 @@ var _ = Describe("Manager", Ordered, func() {
}
Eventually(verifyMetricsServerStarted).Should(Succeed())

By("waiting for webhook service to be ready")
verifyWebhookServiceReady := func(g Gomega) {
const webhookServiceName = "project-webhook-service"

// Webhook service should exist since webhooks are configured
cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
_, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")

// Check if webhook server is ready by verifying pod readiness
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(Equal("True"),
"Controller manager pod not ready (webhook server may not be accepting connections)")

// Check if webhook service endpoints are available
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
output, err = utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")

// Test webhook connectivity by checking if webhook server port is responding
cmd = exec.Command("kubectl", "get", "--raw",
fmt.Sprintf("/api/v1/namespaces/%s/services/https:%s:443/proxy/readyz", namespace, webhookServiceName))
_, err = utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
}
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
// +kubebuilder:scaffold:e2e-webhooks-readiness

By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
"--namespace", namespace,
Expand Down
2 changes: 2 additions & 0 deletions docs/book/src/getting-started/testdata/project/test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,8 @@ var _ = Describe("Manager", Ordered, func() {
}
Eventually(verifyMetricsServerStarted).Should(Succeed())

// +kubebuilder:scaffold:e2e-webhooks-readiness

By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
"--namespace", namespace,
Expand Down
33 changes: 33 additions & 0 deletions docs/book/src/multiversion-tutorial/testdata/project/test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -216,6 +216,39 @@ var _ = Describe("Manager", Ordered, func() {
}
Eventually(verifyMetricsServerStarted).Should(Succeed())

By("waiting for webhook service to be ready")
verifyWebhookServiceReady := func(g Gomega) {
const webhookServiceName = "project-webhook-service"

// Webhook service should exist since webhooks are configured
cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
_, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")

// Check if webhook server is ready by verifying pod readiness
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(Equal("True"),
"Controller manager pod not ready (webhook server may not be accepting connections)")

// Check if webhook service endpoints are available
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
output, err = utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")

// Test webhook connectivity by checking if webhook server port is responding
cmd = exec.Command("kubectl", "get", "--raw",
fmt.Sprintf("/api/v1/namespaces/%s/services/https:%s:443/proxy/readyz", namespace, webhookServiceName))
_, err = utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
}
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
// +kubebuilder:scaffold:e2e-webhooks-readiness

By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
"--namespace", namespace,
Expand Down
1 change: 1 addition & 0 deletions docs/book/src/reference/markers/scaffold.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ properly registered with the manager, so that the controller can reconcile the r
| `+kubebuilder:scaffold:crdkustomizecainjectioname` | `config/default` | Marks where CA injection patches are added for the conversion webhooks. |
| **(No longer supported)** `+kubebuilder:scaffold:crdkustomizecainjectionpatch` | `config/crd` | Marks where CA injection patches are added for the webhooks. Replaced by `+kubebuilder:scaffold:crdkustomizecainjectionns` and `+kubebuilder:scaffold:crdkustomizecainjectioname` |
| `+kubebuilder:scaffold:manifestskustomizesamples` | `config/samples` | Marks where Kustomize sample manifests are injected. |
| `+kubebuilder:scaffold:e2e-webhooks-readiness` | `test/e2e` | Adds webhooks readiness checks in e2e tests before metrics collection. |
| `+kubebuilder:scaffold:e2e-webhooks-checks` | `test/e2e` | Adds e2e checks for webhooks depending on the types of webhooks scaffolded. |

<aside class="note warning">
Expand Down
84 changes: 67 additions & 17 deletions pkg/plugins/golang/v4/scaffolds/internal/templates/test/e2e/test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
log "log/slog"
"os"
"path/filepath"
"strings"

"sigs.k8s.io/kubebuilder/v4/pkg/machinery"
)
Expand All @@ -31,7 +32,10 @@ var (
_ machinery.Inserter = &WebhookTestUpdater{}
)

const webhookChecksMarker = "e2e-webhooks-checks"
const (
webhookChecksMarker = "e2e-webhooks-checks"
webhookReadinessMarker = "e2e-webhooks-readiness"
)

// Test defines the basic setup for the e2e test
type Test struct {
Expand Down Expand Up @@ -74,6 +78,7 @@ func (*WebhookTestUpdater) GetIfExistsAction() machinery.IfExistsAction {
// GetMarkers implements file.Inserter
func (f *WebhookTestUpdater) GetMarkers() []machinery.Marker {
return []machinery.Marker{
machinery.NewMarkerFor(f.GetPath(), webhookReadinessMarker),
machinery.NewMarkerFor(f.GetPath(), webhookChecksMarker),
}
}
Expand Down Expand Up @@ -107,25 +112,35 @@ func (f *WebhookTestUpdater) GetCodeFragments() machinery.CodeFragmentsMap {
}

var fragments []string
fragments = append(fragments, webhookChecksFragment)

if f.Resource != nil && f.Resource.HasDefaultingWebhook() {
mutatingWebhookCode := fmt.Sprintf(mutatingWebhookChecksFragment, f.ProjectName)
fragments = append(fragments, mutatingWebhookCode)
}
// Handle different markers differently
markerString := marker.String()
if strings.Contains(markerString, webhookReadinessMarker) {
// Only inject readiness check for this marker
webhookReadinessCode := fmt.Sprintf(webhookReadinessFragment, f.ProjectName)
fragments = append(fragments, webhookReadinessCode)
} else if strings.Contains(markerString, webhookChecksMarker) {
// Inject all other webhook tests for this marker
fragments = append(fragments, webhookChecksFragment)

if f.Resource != nil && f.Resource.HasDefaultingWebhook() {
mutatingWebhookCode := fmt.Sprintf(mutatingWebhookChecksFragment, f.ProjectName)
fragments = append(fragments, mutatingWebhookCode)
}

if f.Resource != nil && f.Resource.HasValidationWebhook() {
validatingWebhookCode := fmt.Sprintf(validatingWebhookChecksFragment, f.ProjectName)
fragments = append(fragments, validatingWebhookCode)
}
if f.Resource != nil && f.Resource.HasValidationWebhook() {
validatingWebhookCode := fmt.Sprintf(validatingWebhookChecksFragment, f.ProjectName)
fragments = append(fragments, validatingWebhookCode)
}

if f.Resource != nil && f.Resource.HasConversionWebhook() {
conversionWebhookCode := fmt.Sprintf(
conversionWebhookChecksFragment,
f.Resource.Kind,
f.Resource.Plural+"."+f.Resource.Group+"."+f.Resource.Domain,
)
fragments = append(fragments, conversionWebhookCode)
if f.Resource != nil && f.Resource.HasConversionWebhook() {
conversionWebhookCode := fmt.Sprintf(
conversionWebhookChecksFragment,
f.Resource.Kind,
f.Resource.Plural+"."+f.Resource.Group+"."+f.Resource.Domain,
)
fragments = append(fragments, conversionWebhookCode)
}
}

codeFragments[marker] = fragments
Expand Down Expand Up @@ -198,6 +213,39 @@ const conversionWebhookChecksFragment = `It("should have CA injection for %[1]s

`

const webhookReadinessFragment = `By("waiting for webhook service to be ready")
verifyWebhookServiceReady := func(g Gomega) {
const webhookServiceName = "%s-webhook-service"

// Webhook service should exist since webhooks are configured
cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
_, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")

// Check if webhook server is ready by verifying pod readiness
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(Equal("True"),
"Controller manager pod not ready (webhook server may not be accepting connections)")

// Check if webhook service endpoints are available
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
output, err = utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")

// Test webhook connectivity by checking if webhook server port is responding
cmd = exec.Command("kubectl", "get", "--raw",
fmt.Sprintf("/api/v1/namespaces/%%s/services/https:%%s:443/proxy/readyz", namespace, webhookServiceName))
_, err = utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
}
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
`

var testCodeTemplate = `//go:build e2e
// +build e2e

Expand Down Expand Up @@ -394,6 +442,8 @@ var _ = Describe("Manager", Ordered, func() {
}
Eventually(verifyMetricsServerStarted).Should(Succeed())

// +kubebuilder:scaffold:e2e-webhooks-readiness

By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
"--namespace", namespace,
Expand Down
33 changes: 33 additions & 0 deletions testdata/project-v4-multigroup/test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,39 @@ var _ = Describe("Manager", Ordered, func() {
}
Eventually(verifyMetricsServerStarted).Should(Succeed())

By("waiting for webhook service to be ready")
verifyWebhookServiceReady := func(g Gomega) {
const webhookServiceName = "project-v4-multigroup-webhook-service"

// Webhook service should exist since webhooks are configured
cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
_, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")

// Check if webhook server is ready by verifying pod readiness
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(Equal("True"),
"Controller manager pod not ready (webhook server may not be accepting connections)")

// Check if webhook service endpoints are available
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
output, err = utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")

// Test webhook connectivity by checking if webhook server port is responding
cmd = exec.Command("kubectl", "get", "--raw",
fmt.Sprintf("/api/v1/namespaces/%s/services/https:%s:443/proxy/readyz", namespace, webhookServiceName))
_, err = utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
}
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
// +kubebuilder:scaffold:e2e-webhooks-readiness
Copy link
Member

@camilamacedo86 camilamacedo86 Oct 30, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we may can simplify with

By("waiting for webhook service to be ready if webhooks are configured")
verifyWebhookServiceReady := func(g Gomega) {
	const webhookServiceName = "{{ProjectName}}-webhook-service"

	cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
	_, err := utils.Run(cmd)
	if err != nil {
		// Project has no webhook service; nothing to wait on.
		return
	}

	cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
		"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
	output, err := utils.Run(cmd)
	g.Expect(err).NotTo(HaveOccurred())
	g.Expect(output).To(Equal("True"),
		"Controller manager pod not ready (webhook server may not be accepting connections)")

	cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
		"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
	output, err = utils.Run(cmd)
	g.Expect(err).NotTo(HaveOccurred())
	g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")

	cmd = exec.Command("kubectl", "get", "--raw",
		fmt.Sprintf("/api/v1/namespaces/%s/services/https:%[2]s:https/proxy/readyz", namespace, 		webhookServiceName))
	_, err = utils.Run(cmd)
	g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
}
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

WDYT>

Copy link
Contributor Author

@mayuka-c mayuka-c Oct 31, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks you, this looks Perfect :)

Regarding this I have slightly modified. Since now we have the marker which runs exclusively for webhooks, instead of ignoring the error, will be failing if service is not found.

	const webhookServiceName = "{{ProjectName}}-webhook-service"

	cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
	_, err := utils.Run(cmd)
	if err != nil {
		// Project has no webhook service; nothing to wait on.
		return
	}

to

      const webhookServiceName = "{{.ProjectName}}-webhook-service"
      
      // Webhook service should exist since webhooks are configured
      cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
      _, err := utils.Run(cmd)
      g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Regarding this

	cmd = exec.Command("kubectl", "get", "--raw",
		fmt.Sprintf("/api/v1/namespaces/%s/services/https:%[2]s:https/proxy/readyz", namespace, 		webhookServiceName))
	_, err = utils.Run(cmd)
	g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")

Looks like it is failing after using the service proxy to check for readiness here: https://github.com/mayuka-c/kubebuilder/actions/runs/18962257859/job/54151953303.

I will see on what could be the reason for this. Please do let me know if the endpoint used is wrong. Thanks!


By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
"--namespace", namespace,
Expand Down
33 changes: 33 additions & 0 deletions testdata/project-v4-with-plugins/test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,39 @@ var _ = Describe("Manager", Ordered, func() {
}
Eventually(verifyMetricsServerStarted).Should(Succeed())

By("waiting for webhook service to be ready")
verifyWebhookServiceReady := func(g Gomega) {
const webhookServiceName = "project-v4-with-plugins-webhook-service"

// Webhook service should exist since webhooks are configured
cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
_, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")

// Check if webhook server is ready by verifying pod readiness
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(Equal("True"),
"Controller manager pod not ready (webhook server may not be accepting connections)")

// Check if webhook service endpoints are available
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
output, err = utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")

// Test webhook connectivity by checking if webhook server port is responding
cmd = exec.Command("kubectl", "get", "--raw",
fmt.Sprintf("/api/v1/namespaces/%s/services/https:%s:443/proxy/readyz", namespace, webhookServiceName))
_, err = utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
}
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
// +kubebuilder:scaffold:e2e-webhooks-readiness

By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
"--namespace", namespace,
Expand Down
33 changes: 33 additions & 0 deletions testdata/project-v4/test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,39 @@ var _ = Describe("Manager", Ordered, func() {
}
Eventually(verifyMetricsServerStarted).Should(Succeed())

By("waiting for webhook service to be ready")
verifyWebhookServiceReady := func(g Gomega) {
const webhookServiceName = "project-v4-webhook-service"

// Webhook service should exist since webhooks are configured
cmd := exec.Command("kubectl", "get", "service", webhookServiceName, "-n", namespace)
_, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook service should exist but was not found")

// Check if webhook server is ready by verifying pod readiness
cmd = exec.Command("kubectl", "get", "pods", "-l", "control-plane=controller-manager",
"-n", namespace, "-o", "jsonpath={.items[0].status.conditions[?(@.type=='Ready')].status}")
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(Equal("True"),
"Controller manager pod not ready (webhook server may not be accepting connections)")

// Check if webhook service endpoints are available
cmd = exec.Command("kubectl", "get", "endpoints", webhookServiceName,
"-n", namespace, "-o", "jsonpath={.subsets[*].addresses[*].ip}")
output, err = utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).NotTo(BeEmpty(), "Webhook service endpoints are not ready")

// Test webhook connectivity by checking if webhook server port is responding
cmd = exec.Command("kubectl", "get", "--raw",
fmt.Sprintf("/api/v1/namespaces/%s/services/https:%s:443/proxy/readyz", namespace, webhookServiceName))
_, err = utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Webhook server not responding on port 443")
}
Eventually(verifyWebhookServiceReady, 2*time.Minute).Should(Succeed())
// +kubebuilder:scaffold:e2e-webhooks-readiness

By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
"--namespace", namespace,
Expand Down