Skip to content

Commit e1cb1a7

Browse files
EtiennePerotgvisor-bot
EtiennePerot
authored and
gvisor-bot
committed
testcluster: Remove all dependencies on cluster spec protobuffer.
Instead, cluster nodepools are reconstructed from Kubernetes client API data and looking at their labels from that. Then these are used for configuring pods to schedule on these nodes. This reduces dependencies on GKE-specific implementation details and makes benchmarks easier to run outside of GKE. PiperOrigin-RevId: 700447594
1 parent 1af6da4 commit e1cb1a7

File tree

20 files changed

+335
-208
lines changed

20 files changed

+335
-208
lines changed

‎test/kubernetes/benchmarks/abslbuild.go

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,11 @@ func BuildABSL(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex
5353
defer cluster.DeletePersistentVolume(ctx, persistentVol)
5454

5555
image := imageAMD
56-
if cluster.RuntimeTestNodepoolIsARM() {
56+
testCPUArch, err := cluster.RuntimeTestNodepoolArchitecture(ctx)
57+
if err != nil {
58+
t.Fatalf("Failed to get runtime test nodepool architecture: %v", err)
59+
}
60+
if testCPUArch == testcluster.CPUArchitectureARM {
5761
t.Skipf("Building ABSL is not supported on ARM")
5862
return
5963
}
@@ -91,14 +95,14 @@ func BuildABSL(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex
9195
},
9296
} {
9397
t.Run(test.name, func(t *testing.T) {
94-
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
98+
endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS)
9599
if err != nil {
96100
t.Fatalf("Failed to setup profiling: %v", err)
97101
}
98102
defer endProfiling()
99103

100104
pod := newABSLPod(benchmarkNS, name, image, test.volume)
101-
pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod)
105+
pod, err = cluster.ConfigurePodForRuntimeTestNodepool(ctx, pod)
102106
if err != nil {
103107
t.Fatalf("Failed to set pod for test runtime: %v", err)
104108
}

‎test/kubernetes/benchmarks/ffmpeg.go

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,18 @@ func RunFFMPEG(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex
5353
}
5454
defer cluster.DeletePersistentVolume(ctx, persistentVol)
5555

56-
image := imageAMD
57-
if cluster.RuntimeTestNodepoolIsARM() {
56+
testCPUArch, err := cluster.RuntimeTestNodepoolArchitecture(ctx)
57+
if err != nil {
58+
t.Fatalf("Failed to get runtime test nodepool architecture: %v", err)
59+
}
60+
var image string
61+
switch testCPUArch {
62+
case testcluster.CPUArchitectureX86:
63+
image = imageAMD
64+
case testcluster.CPUArchitectureARM:
5865
image = imageARM
66+
default:
67+
t.Fatalf("Unsupported CPU architecture: %v", testCPUArch)
5968
}
6069
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
6170
t.Fatalf("Failed to resolve image: %v", err)
@@ -91,14 +100,14 @@ func RunFFMPEG(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex
91100
},
92101
} {
93102
t.Run(test.name, func(t *testing.T) {
94-
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
103+
endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS)
95104
if err != nil {
96105
t.Fatalf("Failed to setup profiling: %v", err)
97106
}
98107
defer endProfiling()
99108

100109
p := newFfmpegDevPod(benchmarkNS, name, image, test.volume)
101-
p, err = cluster.ConfigurePodForRuntimeTestNodepool(p)
110+
p, err = cluster.ConfigurePodForRuntimeTestNodepool(ctx, p)
102111
if err != nil {
103112
t.Fatalf("Failed to configure pod for runtime: %v", err)
104113
}

‎test/kubernetes/benchmarks/grpc.go

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,18 @@ func BuildGRPC(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex
5353
}
5454
defer cluster.DeletePersistentVolume(ctx, persistentVol)
5555

56-
image := imageAMD
57-
if cluster.RuntimeTestNodepoolIsARM() {
56+
testCPUArch, err := cluster.RuntimeTestNodepoolArchitecture(ctx)
57+
if err != nil {
58+
t.Fatalf("Failed to get runtime test nodepool architecture: %v", err)
59+
}
60+
var image string
61+
switch testCPUArch {
62+
case testcluster.CPUArchitectureX86:
63+
image = imageAMD
64+
case testcluster.CPUArchitectureARM:
5865
image = imageARM
66+
default:
67+
t.Fatalf("Unsupported CPU architecture: %v", testCPUArch)
5968
}
6069
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
6170
t.Fatalf("Failed to resolve image: %v", err)
@@ -91,14 +100,14 @@ func BuildGRPC(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex
91100
},
92101
} {
93102
t.Run(test.name, func(t *testing.T) {
94-
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
103+
endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS)
95104
if err != nil {
96105
t.Fatalf("Failed to setup profiling: %v", err)
97106
}
98107
defer endProfiling()
99108

100109
pod := newGRPCPod(benchmarkNS, name, image, test.volume)
101-
pod, err = cluster.ConfigurePodForRuntimeTestNodepool(pod)
110+
pod, err = cluster.ConfigurePodForRuntimeTestNodepool(ctx, pod)
102111
if err != nil {
103112
t.Fatalf("Failed to set pod for test runtime: %v", err)
104113
}

‎test/kubernetes/benchmarks/gsutil.go

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,18 @@ func RunGSUtil(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex
5656
}
5757
defer cluster.DeletePersistentVolume(ctx, persistentVol)
5858

59-
image := imageAMD
60-
if cluster.RuntimeTestNodepoolIsARM() {
59+
testCPUArch, err := cluster.RuntimeTestNodepoolArchitecture(ctx)
60+
if err != nil {
61+
t.Fatalf("Failed to get runtime test nodepool architecture: %v", err)
62+
}
63+
var image string
64+
switch testCPUArch {
65+
case testcluster.CPUArchitectureX86:
66+
image = imageAMD
67+
case testcluster.CPUArchitectureARM:
6168
image = imageARM
69+
default:
70+
t.Fatalf("Unsupported CPU architecture: %v", testCPUArch)
6271
}
6372
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
6473
t.Fatalf("Failed to resolve image: %v", err)
@@ -113,7 +122,7 @@ func RunGSUtil(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex
113122
} {
114123
t.Run(slicing.name, func(t *testing.T) {
115124
// Setup profiling if requested by the user.
116-
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
125+
endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS)
117126
if err != nil {
118127
t.Fatalf("Failed to setup profiling: %v", err)
119128
}
@@ -122,7 +131,7 @@ func RunGSUtil(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContex
122131
// Create a pod that performs setup, then times
123132
// downloading.
124133
p := newGSUtilDevPod(benchmarkNS, name, image, storage.volume, slicing.option)
125-
p, err = cluster.ConfigurePodForRuntimeTestNodepool(p)
134+
p, err = cluster.ConfigurePodForRuntimeTestNodepool(ctx, p)
126135
if err != nil {
127136
t.Fatalf("Failed to configure pod for runtime: %v", err)
128137
}

‎test/kubernetes/benchmarks/httpbench/httpbench.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ func (h *HTTPBenchmark) runRound(ctx context.Context, t *testing.T, round Round,
135135
}
136136
name := fmt.Sprintf("wrk2-%dthreads-%sqps", round.NumThreads, qpsText)
137137
client := h.newWrk2Client(name, ip, round)
138-
client, err := h.Cluster.ConfigurePodForClientNodepool(client)
138+
client, err := h.Cluster.ConfigurePodForClientNodepool(ctx, client)
139139
if err != nil {
140140
t.Fatalf("failed to configure wrk2 pod for client nodepool: %v", err)
141141
}
@@ -243,7 +243,7 @@ func (h *HTTPBenchmark) getWgetPod(ip string) *v13.Pod {
243243
// waitForServer waits for an HTTP server to start responding on the given
244244
// IP and port.
245245
func (h *HTTPBenchmark) waitForServer(ctx context.Context, ip string) error {
246-
wget, err := h.Cluster.ConfigurePodForClientNodepool(h.getWgetPod(ip))
246+
wget, err := h.Cluster.ConfigurePodForClientNodepool(ctx, h.getWgetPod(ip))
247247
if err != nil {
248248
return fmt.Errorf("failed to configure wget pod for client nodepool: %v", err)
249249
}

‎test/kubernetes/benchmarks/nginx.go

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,20 @@ func BenchmarkNginx(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesC
6262
}
6363
defer benchmarkNS.Cleanup(ctx)
6464

65-
nginxImage := nginxImageAMD
66-
if cluster.RuntimeTestNodepoolIsARM() {
65+
testCPUArch, err := cluster.RuntimeTestNodepoolArchitecture(ctx)
66+
if err != nil {
67+
t.Fatalf("Failed to get runtime test nodepool architecture: %v", err)
68+
}
69+
var nginxImage string
70+
switch testCPUArch {
71+
case testcluster.CPUArchitectureX86:
72+
nginxImage = nginxImageAMD
73+
case testcluster.CPUArchitectureARM:
6774
nginxImage = nginxImageARM
75+
default:
76+
t.Fatalf("Unsupported CPU architecture: %v", testCPUArch)
6877
}
69-
nginxImage, err := k8sCtx.ResolveImage(ctx, nginxImage)
70-
if err != nil {
78+
if nginxImage, err = k8sCtx.ResolveImage(ctx, nginxImage); err != nil {
7179
t.Fatalf("Failed to resolve image: %v", err)
7280
}
7381

@@ -114,7 +122,7 @@ func BenchmarkNginx(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesC
114122
},
115123
} {
116124
t.Run(test.name, func(t *testing.T) {
117-
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
125+
endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS)
118126
if err != nil {
119127
t.Fatalf("Failed to setup profiling: %v", err)
120128
}
@@ -123,7 +131,7 @@ func BenchmarkNginx(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesC
123131
name := fmt.Sprintf("nginx-%s", test.suffix)
124132

125133
server := newNginxServer(benchmarkNS, name, nginxImage, test.volume)
126-
server, err = cluster.ConfigurePodForRuntimeTestNodepool(server)
134+
server, err = cluster.ConfigurePodForRuntimeTestNodepool(ctx, server)
127135
if err != nil {
128136
t.Fatalf("Failed to configure pod for runtime nodepool: %v", err)
129137
}

‎test/kubernetes/benchmarks/ollama.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ func (ops *ollamaPodServer) InstrumentedRequest(ctx context.Context, argvFn func
164164
RestartPolicy: v13.RestartPolicyNever,
165165
},
166166
}
167-
clientPod, err := ops.cluster.ConfigurePodForClientNodepool(clientPod)
167+
clientPod, err := ops.cluster.ConfigurePodForClientNodepool(ctx, clientPod)
168168
if err != nil {
169169
return nil, fmt.Errorf("failed to configure pod: %v", err)
170170
}
@@ -245,7 +245,7 @@ func BenchmarkOllama(ctx context.Context, t *testing.T, k8sCtx k8sctx.Kubernetes
245245
if err != nil {
246246
t.Fatalf("Failed to resolve image: %v", err)
247247
}
248-
ollamaPod, err := cluster.ConfigurePodForRuntimeTestNodepool(newOllamaServerPod(benchmarkNS, serverImage))
248+
ollamaPod, err := cluster.ConfigurePodForRuntimeTestNodepool(ctx, newOllamaServerPod(benchmarkNS, serverImage))
249249
if err != nil {
250250
t.Fatalf("Failed to configure pod for runtime nodepool: %v", err)
251251
}

‎test/kubernetes/benchmarks/postgresql.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ func BenchmarkPostgresPGBench(ctx context.Context, t *testing.T, k8sCtx k8sctx.K
5656
t.Fatalf("cannot reset namespace: %v", err)
5757
}
5858
defer benchmarkNS.Cleanup(ctx)
59-
endProfiling, err := profiling.MaybeSetup(ctx, t, cluster, benchmarkNS)
59+
endProfiling, err := profiling.MaybeSetup(ctx, t, k8sCtx, cluster, benchmarkNS)
6060
if err != nil {
6161
t.Fatalf("Failed to setup profiling: %v", err)
6262
}
@@ -80,7 +80,7 @@ func BenchmarkPostgresPGBench(ctx context.Context, t *testing.T, k8sCtx k8sctx.K
8080
server.ObjectMeta.Labels = make(map[string]string)
8181
}
8282
server.ObjectMeta.Labels[postgresServerLabelKey] = postgresServerLabelValue
83-
server, err = cluster.ConfigurePodForRuntimeTestNodepool(server)
83+
server, err = cluster.ConfigurePodForRuntimeTestNodepool(ctx, server)
8484
if err != nil {
8585
t.Fatalf("ConfigurePodForRuntimeTestNodepool on cluster %q: %v", cluster.GetName(), err)
8686
}
@@ -127,7 +127,7 @@ func BenchmarkPostgresPGBench(ctx context.Context, t *testing.T, k8sCtx k8sctx.K
127127
fmt.Sprintf("--username=%s", postgresUser),
128128
fmt.Sprintf("--dbname=%s", postgresDatabase),
129129
}, false /* withPort */, nil /* pvc */)
130-
pgIsReady, err = cluster.ConfigurePodForClientNodepool(pgIsReady)
130+
pgIsReady, err = cluster.ConfigurePodForClientNodepool(ctx, pgIsReady)
131131
if err != nil {
132132
return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), pgIsReadyName, err)
133133
}
@@ -170,7 +170,7 @@ func BenchmarkPostgresPGBench(ctx context.Context, t *testing.T, k8sCtx k8sctx.K
170170
fmt.Sprintf("--username=%s", postgresUser),
171171
postgresDatabase,
172172
}, false /* withPort */, nil /* pvc */)
173-
initDB, err = cluster.ConfigurePodForClientNodepool(initDB)
173+
initDB, err = cluster.ConfigurePodForClientNodepool(ctx, initDB)
174174
if err != nil {
175175
return fmt.Errorf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), initDBName, err)
176176
}
@@ -206,7 +206,7 @@ func BenchmarkPostgresPGBench(ctx context.Context, t *testing.T, k8sCtx k8sctx.K
206206
postgresDatabase,
207207
}
208208
client := newPostgresPod(benchmarkNS, "pgbench", image, clientCmd, false /* withPort */, nil /* pvc */)
209-
client, err = cluster.ConfigurePodForClientNodepool(client)
209+
client, err = cluster.ConfigurePodForClientNodepool(ctx, client)
210210
if err != nil {
211211
t.Fatalf("ConfigurePodForClientNodepool on cluster %q: pod: %q: %v", cluster.GetName(), client.GetName(), err)
212212
}

‎test/kubernetes/benchmarks/profiling/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ go_library(
1717
"//pkg/atomicbitops",
1818
"//runsc/flag",
1919
"//test/kubernetes",
20+
"//test/kubernetes/k8sctx",
2021
"//test/kubernetes/testcluster",
2122
"//test/metricsviz",
2223
"@com_github_google_pprof//profile:go_default_library",

‎test/kubernetes/benchmarks/profiling/profiling.go

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343
"gvisor.dev/gvisor/pkg/atomicbitops"
4444
"gvisor.dev/gvisor/runsc/flag"
4545
k8s "gvisor.dev/gvisor/test/kubernetes"
46+
"gvisor.dev/gvisor/test/kubernetes/k8sctx"
4647
"gvisor.dev/gvisor/test/kubernetes/testcluster"
4748
"gvisor.dev/gvisor/test/metricsviz"
4849
appsv1 "k8s.io/api/apps/v1"
@@ -203,17 +204,29 @@ func streamDir(dirPath string) operation {
203204
}
204205

205206
// startsOperations starts the given operations in a DaemonSet.
206-
func startOperations(ctx context.Context, c *testcluster.TestCluster, ns *testcluster.Namespace, operations []operation) (*appsv1.DaemonSet, error) {
207+
func startOperations(ctx context.Context, k8sCtx k8sctx.KubernetesContext, c *testcluster.TestCluster, ns *testcluster.Namespace, operations []operation) (*appsv1.DaemonSet, error) {
207208
ds := profileDSTemplate(c)
208209
ds.Namespace = ns.Namespace
209210
ds.ObjectMeta.Namespace = ns.Namespace
210211
ds.Spec.Template.Namespace = ns.Namespace
211212
ds.Spec.Template.ObjectMeta.Namespace = ns.Namespace
212-
c.ConfigureDaemonSetForRuntimeTestNodepool(&ds)
213+
c.ConfigureDaemonSetForRuntimeTestNodepool(ctx, &ds)
213214
ds.Spec.Template.Spec.RuntimeClassName = nil // Must run unsandboxed.
214-
image := profileHelperImageAMD64
215-
if c.RuntimeTestNodepoolIsARM() {
215+
var image string
216+
testCPUArch, err := c.RuntimeTestNodepoolArchitecture(ctx)
217+
if err != nil {
218+
return nil, fmt.Errorf("failed to determine test CPU architecture: %w", err)
219+
}
220+
switch testCPUArch {
221+
case testcluster.CPUArchitectureX86:
222+
image = profileHelperImageAMD64
223+
case testcluster.CPUArchitectureARM:
216224
image = profileHelperImageARM64
225+
default:
226+
return nil, fmt.Errorf("unsupported CPU architecture: %q", testCPUArch)
227+
}
228+
if image, err = k8sCtx.ResolveImage(ctx, image); err != nil {
229+
return nil, fmt.Errorf("failed to resolve image %q: %w", image, err)
217230
}
218231
for i, op := range operations {
219232
name := op.name
@@ -247,6 +260,7 @@ func startOperations(ctx context.Context, c *testcluster.TestCluster, ns *testcl
247260
// profileRun encapsulates data about a profiling run.
248261
// It is used after the run completes so that profiles can be retrieved.
249262
type profileRun struct {
263+
k8sCtx k8sctx.KubernetesContext
250264
c *testcluster.TestCluster
251265
ns *testcluster.Namespace
252266
localProfileDir string
@@ -256,7 +270,7 @@ type profileRun struct {
256270
// MaybeSetup sets up profiling if requested. It returns a cleanup function.
257271
// If the returned error is nil, the cleanup function is non-nil and should be
258272
// called regardless of whether profiling is actually enabled or not.
259-
func MaybeSetup(ctx context.Context, t *testing.T, c *testcluster.TestCluster, ns *testcluster.Namespace) (func(), error) {
273+
func MaybeSetup(ctx context.Context, t *testing.T, k8sCtx k8sctx.KubernetesContext, c *testcluster.TestCluster, ns *testcluster.Namespace) (func(), error) {
260274
profileDirName := fmt.Sprintf("%s.%s", t.Name(), time.Now().Format("20060102-150405"))
261275
profileDirName = regexp.MustCompile("[^-_=.\\w]+").ReplaceAllString(profileDirName, ".")
262276
hasGVisorRuntime, err := c.HasGVisorTestRuntime(ctx)
@@ -316,6 +330,7 @@ func MaybeSetup(ctx context.Context, t *testing.T, c *testcluster.TestCluster, n
316330
}
317331
cleanup = func() {
318332
err := processProfileRun(ctx, t, &profileRun{
333+
k8sCtx: k8sCtx,
319334
c: c,
320335
ns: ns,
321336
localProfileDir: localProfileDir,
@@ -340,7 +355,7 @@ func MaybeSetup(ctx context.Context, t *testing.T, c *testcluster.TestCluster, n
340355
if len(setupCommands) > 0 {
341356
setupCtx, setupCancel := context.WithTimeout(ctx, 2*time.Minute)
342357
defer setupCancel()
343-
ds, err := startOperations(setupCtx, c, ns, setupCommands)
358+
ds, err := startOperations(setupCtx, k8sCtx, c, ns, setupCommands)
344359
if err != nil {
345360
return nil, err
346361
}
@@ -359,7 +374,7 @@ func processProfileRun(ctx context.Context, t *testing.T, run *profileRun) error
359374
beforeSpawn := metav1.NewTime(time.Now())
360375
retrievalCtx, retrievalCancel := context.WithCancel(ctx)
361376
defer retrievalCancel()
362-
ds, err := startOperations(retrievalCtx, run.c, run.ns, []operation{
377+
ds, err := startOperations(retrievalCtx, run.k8sCtx, run.c, run.ns, []operation{
363378
dirOp,
364379
setFlag("profile", "false"),
365380
removeFlag("profile-cpu"),

0 commit comments

Comments
 (0)