diff --git a/test/kubernetes/benchmarks/BUILD b/test/kubernetes/benchmarks/BUILD index 901831d6b5..5d2fd2ea3b 100644 --- a/test/kubernetes/benchmarks/BUILD +++ b/test/kubernetes/benchmarks/BUILD @@ -58,7 +58,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -89,7 +89,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -122,7 +122,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -155,7 +155,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -188,7 +188,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -222,7 +222,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -254,7 +254,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -287,7 +287,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -320,7 +320,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -352,7 +352,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -392,7 +392,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -426,7 +426,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -458,7 +458,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) @@ -491,7 +491,7 @@ go_test( ], deps = [ "//test/kubernetes/k8sctx", - "//test/kubernetes/k8sctx/autok8sctx", + "//test/kubernetes/k8sctx/kubectlctx", "//test/kubernetes/testcluster", ], ) diff --git a/test/kubernetes/benchmarks/abslbuild_test.go b/test/kubernetes/benchmarks/abslbuild_test.go index 7995023efc..2095789b00 100644 --- a/test/kubernetes/benchmarks/abslbuild_test.go +++ b/test/kubernetes/benchmarks/abslbuild_test.go @@ -19,14 +19,14 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) // TestABSLBuild benchmarks building various Abseil C++ targets. func TestABSLBuild(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/ffmpeg_test.go b/test/kubernetes/benchmarks/ffmpeg_test.go index 9b9ebe3c5e..ec00cb10e1 100644 --- a/test/kubernetes/benchmarks/ffmpeg_test.go +++ b/test/kubernetes/benchmarks/ffmpeg_test.go @@ -19,13 +19,13 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) func TestFfmpeg(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/grpc_test.go b/test/kubernetes/benchmarks/grpc_test.go index fab0a4ff9d..023ca33d87 100644 --- a/test/kubernetes/benchmarks/grpc_test.go +++ b/test/kubernetes/benchmarks/grpc_test.go @@ -19,13 +19,13 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) func TestGRPCBuild(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/gsutil_test.go b/test/kubernetes/benchmarks/gsutil_test.go index 66e67d366d..ba687dcd61 100644 --- a/test/kubernetes/benchmarks/gsutil_test.go +++ b/test/kubernetes/benchmarks/gsutil_test.go @@ -21,13 +21,13 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) func TestGSUtil(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/nginx_test.go b/test/kubernetes/benchmarks/nginx_test.go index adc55c702c..28c9d56c32 100644 --- a/test/kubernetes/benchmarks/nginx_test.go +++ b/test/kubernetes/benchmarks/nginx_test.go @@ -19,13 +19,13 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) func TestNginx(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/ollama_test.go b/test/kubernetes/benchmarks/ollama_test.go index 929335f114..8dd7205c11 100644 --- a/test/kubernetes/benchmarks/ollama_test.go +++ b/test/kubernetes/benchmarks/ollama_test.go @@ -22,7 +22,7 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) @@ -30,7 +30,7 @@ func TestOllama(t *testing.T) { fmt.Fprint(os.Stderr, "HEADS UP: This test uses a huge container image which may take up to 30 minutes to download onto nodes the first time you run it.\n") ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/postgresql_test.go b/test/kubernetes/benchmarks/postgresql_test.go index 1645d744dd..4062eca4e9 100644 --- a/test/kubernetes/benchmarks/postgresql_test.go +++ b/test/kubernetes/benchmarks/postgresql_test.go @@ -20,14 +20,14 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) // TestPostgresPGBench benchmarks a PostgreSQL database with pgbench. func TestPostgresPGBench(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/pytorch_test.go b/test/kubernetes/benchmarks/pytorch_test.go index 7879f5c4a6..1b1f37513e 100644 --- a/test/kubernetes/benchmarks/pytorch_test.go +++ b/test/kubernetes/benchmarks/pytorch_test.go @@ -19,7 +19,7 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) @@ -49,7 +49,7 @@ func TestMobileNetV2(t *testing.T) { } func runTests(ctx context.Context, t *testing.T, tests []pytorchTest) { - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/redis_test.go b/test/kubernetes/benchmarks/redis_test.go index 3e01ccdf39..9561b3459f 100644 --- a/test/kubernetes/benchmarks/redis_test.go +++ b/test/kubernetes/benchmarks/redis_test.go @@ -19,14 +19,14 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) // TestRedis benchmarks redis servers on k8s clusters. func TestRedis(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/rubydev_test.go b/test/kubernetes/benchmarks/rubydev_test.go index b5da72e914..4c8ed0a011 100644 --- a/test/kubernetes/benchmarks/rubydev_test.go +++ b/test/kubernetes/benchmarks/rubydev_test.go @@ -19,14 +19,14 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) // TestRubyDev benchmarks a build job on k8s clusters. func TestRubyDev(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/stablediffusion_test.go b/test/kubernetes/benchmarks/stablediffusion_test.go index d78ef3326d..625da4a132 100644 --- a/test/kubernetes/benchmarks/stablediffusion_test.go +++ b/test/kubernetes/benchmarks/stablediffusion_test.go @@ -19,13 +19,13 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) func TestStableDiffusionXL(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/startup_test.go b/test/kubernetes/benchmarks/startup_test.go index 9ea889fd5b..f0c378d261 100644 --- a/test/kubernetes/benchmarks/startup_test.go +++ b/test/kubernetes/benchmarks/startup_test.go @@ -19,13 +19,13 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) func TestStartup(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/tensorflow_test.go b/test/kubernetes/benchmarks/tensorflow_test.go index 2161c1b732..80e2d21f79 100644 --- a/test/kubernetes/benchmarks/tensorflow_test.go +++ b/test/kubernetes/benchmarks/tensorflow_test.go @@ -19,13 +19,13 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) func TestTensorflowOnCPU(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/benchmarks/wordpress_test.go b/test/kubernetes/benchmarks/wordpress_test.go index 88f574d84b..7338fb6744 100644 --- a/test/kubernetes/benchmarks/wordpress_test.go +++ b/test/kubernetes/benchmarks/wordpress_test.go @@ -19,13 +19,13 @@ import ( "testing" "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" "gvisor.dev/gvisor/test/kubernetes/testcluster" ) func TestWordpress(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) } diff --git a/test/kubernetes/k8sctx/BUILD b/test/kubernetes/k8sctx/BUILD index 5f8137af31..66ea639b6a 100644 --- a/test/kubernetes/k8sctx/BUILD +++ b/test/kubernetes/k8sctx/BUILD @@ -7,7 +7,6 @@ package( go_library( name = "k8sctx", - testonly = True, srcs = [ "k8sctx.go", ], diff --git a/test/kubernetes/k8sctx/autok8sctx/kubectlctx.go b/test/kubernetes/k8sctx/autok8sctx/kubectlctx.go deleted file mode 100644 index 6a0ab92857..0000000000 --- a/test/kubernetes/k8sctx/autok8sctx/kubectlctx.go +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2024 The gVisor Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//go:build !false -// +build !false - -// Package autok8sctx provides a KubernetesContext that uses a kubectl config -// and context to determine the test cluster to use for tests and benchmarks. -// See parent package (`k8sctx`) for more info. -package autok8sctx - -import ( - "context" - "errors" - "fmt" - - "gvisor.dev/gvisor/runsc/flag" - "gvisor.dev/gvisor/test/kubernetes/k8sctx" - "gvisor.dev/gvisor/test/kubernetes/testcluster" - "gvisor.dev/gvisor/tools/gvisor_k8s_tool/provider/kubectl" -) - -var ( - kubectlContextName = flag.String("kubectl-context-name", "", "Name of the kubectl context to use within the kubectl config") -) - -// New creates a KubernetesContext using flags to determine which kubectl -// config and kubectl context to use as the test cluster. -func New(ctx context.Context) (k8sctx.KubernetesContext, error) { - if *kubectlContextName == "" { - return nil, errors.New("no kubectl context name specified") - } - cluster, err := kubectl.NewCluster(*kubectlContextName) - if err != nil { - return nil, fmt.Errorf("cannot initialize cluster %q: %w", *kubectlContextName, err) - } - testCluster := testcluster.NewTestClusterFromClient(*kubectlContextName, cluster.Client()) - return k8sctx.NewSingleCluster(testCluster), nil -} diff --git a/test/kubernetes/k8sctx/k8sctx.go b/test/kubernetes/k8sctx/k8sctx.go index bbcefbc074..338a5b59f1 100644 --- a/test/kubernetes/k8sctx/k8sctx.go +++ b/test/kubernetes/k8sctx/k8sctx.go @@ -21,7 +21,6 @@ package k8sctx import ( "context" - "sync" "testing" "gvisor.dev/gvisor/test/kubernetes/testcluster" @@ -66,34 +65,33 @@ func ForEachCluster(ctx context.Context, t *testing.T, k8sCtx KubernetesContext, } } -// singleCluster implements KubernetesContext using a single cluster. -type singleCluster struct { - mu sync.Mutex - cluster *testcluster.TestCluster +// clusters implements KubernetesContext using a set of clusters. +type clusters struct { + clustersCh chan *testcluster.TestCluster } // Cluster implements KubernetesContext.Cluster. -func (sc *singleCluster) Cluster(ctx context.Context, t *testing.T) (*testcluster.TestCluster, func()) { - sc.mu.Lock() - defer sc.mu.Unlock() - cl := sc.cluster - sc.cluster = nil - return cl, func() { - if cl != nil { - sc.mu.Lock() - defer sc.mu.Unlock() - sc.cluster = cl +func (cs *clusters) Cluster(ctx context.Context, t *testing.T) (*testcluster.TestCluster, func()) { + select { + case cluster := <-cs.clustersCh: + return cluster, func() { + cs.clustersCh <- cluster } + default: + return nil, func() {} } } // ResolveImage implements KubernetesContext.ResolveImage. -func (*singleCluster) ResolveImage(ctx context.Context, imageName string) (string, error) { +func (*clusters) ResolveImage(ctx context.Context, imageName string) (string, error) { return imageName, nil } -// NewSingleCluster creates a KubernetesContext that uses a single, static -// test cluster. -func NewSingleCluster(cluster *testcluster.TestCluster) KubernetesContext { - return &singleCluster{cluster: cluster} +// New creates a KubernetesContext that set of test clusters. +func New(testClusters ...*testcluster.TestCluster) KubernetesContext { + clustersCh := make(chan *testcluster.TestCluster, len(testClusters)) + for _, cluster := range testClusters { + clustersCh <- cluster + } + return &clusters{clustersCh: clustersCh} } diff --git a/test/kubernetes/k8sctx/autok8sctx/BUILD b/test/kubernetes/k8sctx/kubectlctx/BUILD similarity index 55% rename from test/kubernetes/k8sctx/autok8sctx/BUILD rename to test/kubernetes/k8sctx/kubectlctx/BUILD index 7d005b66db..6675af376a 100644 --- a/test/kubernetes/k8sctx/autok8sctx/BUILD +++ b/test/kubernetes/k8sctx/kubectlctx/BUILD @@ -6,8 +6,7 @@ package( ) go_library( - name = "autok8sctx", - testonly = True, + name = "kubectlctx", srcs = ["kubectlctx.go"], nogo = False, visibility = [ @@ -15,8 +14,13 @@ go_library( ], deps = [ "//runsc/flag", + "//test/kubernetes:test_range_config_go_proto", "//test/kubernetes/k8sctx", "//test/kubernetes/testcluster", "//tools/gvisor_k8s_tool/provider/kubectl", + "@io_k8s_client_go//kubernetes:go_default_library", + "@io_k8s_client_go//tools/clientcmd:go_default_library", + "@org_golang_google_protobuf//encoding/prototext:go_default_library", + "@org_golang_x_sync//errgroup:go_default_library", ], ) diff --git a/test/kubernetes/k8sctx/kubectlctx/kubectlctx.go b/test/kubernetes/k8sctx/kubectlctx/kubectlctx.go new file mode 100644 index 0000000000..f2288de8b2 --- /dev/null +++ b/test/kubernetes/k8sctx/kubectlctx/kubectlctx.go @@ -0,0 +1,142 @@ +// Copyright 2024 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package kubectlctx provides a KubernetesContext that uses one or more +// kubectl configs to determine the cluster(s) to use for tests and benchmarks. +// See parent package (`k8sctx`) for more info. +package kubectlctx + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/encoding/prototext" + "gvisor.dev/gvisor/runsc/flag" + "gvisor.dev/gvisor/test/kubernetes/k8sctx" + testpb "gvisor.dev/gvisor/test/kubernetes/test_range_config_go_proto" + "gvisor.dev/gvisor/test/kubernetes/testcluster" + "gvisor.dev/gvisor/tools/gvisor_k8s_tool/provider/kubectl" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +var ( + rangeDir = flag.String("range-dir", "", "A directory containing a test_range.textproto text file describing multiple clusters to use for tests and benchmarks; takes precedence over --kubectl-context-name") + kubectlContext = flag.String("kubectl-context", "", "The name of the kubectl context to use; if unset, use the default context within the kubectl config at KUBECONFIG") + testNodepoolRuntime = flag.String("test-nodepool-runtime", "", "if set, override the runtime used for pods scheduled on the 'test' nodepool. If unset, the nodepool default is used") +) + +// New creates a KubernetesContext using flags to determine which clusters +// to use for tests and benchmarks. +func New(ctx context.Context) (k8sctx.KubernetesContext, error) { + if *rangeDir != "" && *kubectlContext != "" { + return nil, fmt.Errorf("cannot use --range-dir and --kubectl-context at the same time") + } + var clusters []*testcluster.TestCluster + var err error + if *rangeDir != "" { + clusters, err = NewFromRangeDir(ctx, *rangeDir) + } else { + clusters, err = NewFromKubectlContext(ctx, *kubectlContext) + } + if err != nil { + return nil, fmt.Errorf("cannot initialize test clusters: %w", err) + } + if *testNodepoolRuntime != "" { + overriddenRuntime := testcluster.RuntimeType(*testNodepoolRuntime) + if !overriddenRuntime.IsValid() { + return nil, fmt.Errorf("invalid runtime type %q", *testNodepoolRuntime) + } + for _, cluster := range clusters { + cluster.OverrideTestNodepoolRuntime(overriddenRuntime) + } + } + if err := verifyClusters(ctx, clusters); err != nil { + return nil, fmt.Errorf("cannot verify clusters are working: %w", err) + } + return k8sctx.New(clusters...), nil +} + +// NewFromRangeDir creates a set of test clusters from a test range directory. +func NewFromRangeDir(ctx context.Context, rangeDir string) ([]*testcluster.TestCluster, error) { + rangeFile := filepath.Join(rangeDir, "test_range.textproto") + rangeFileData, err := os.ReadFile(rangeFile) + if err != nil { + return nil, fmt.Errorf("cannot read range file %q: %w", rangeFile, err) + } + var testRange testpb.TestRange + if err := prototext.Unmarshal(rangeFileData, &testRange); err != nil { + return nil, fmt.Errorf("error unmarshalling range file %q: %v", rangeFile, err) + } + if len(testRange.GetClusters()) == 0 { + return nil, fmt.Errorf("range file %q has no clusters", rangeFile) + } + clusters := make([]*testcluster.TestCluster, len(testRange.GetClusters())) + for i, cluster := range testRange.GetClusters() { + configPath := cluster.GetKubectlConfig() + if configPath == "" { + return nil, fmt.Errorf("cluster %q has no kubectl config path", cluster.GetCluster()) + } + cfg, err := clientcmd.LoadFromFile(configPath) + if err != nil { + return nil, fmt.Errorf("cannot load kubectl config at %q for cluster %q: %w", configPath, cluster.GetCluster(), err) + } + contextName := cluster.GetKubectlContext() + if contextName == "" { + contextName = cfg.CurrentContext + } + restConfig, err := clientcmd.NewNonInteractiveClientConfig(*cfg, contextName, nil, clientcmd.NewDefaultClientConfigLoadingRules()).ClientConfig() + if err != nil { + return nil, fmt.Errorf("cannot load REST client config for cluster %q: %w", cluster.GetCluster(), err) + } + kubeClient, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, fmt.Errorf("cannot create Kubernetes client for cluster %q: %w", cluster.GetCluster(), err) + } + clusters[i] = testcluster.NewTestClusterFromClient(cluster.GetCluster(), kubeClient) + } + return clusters, nil +} + +// NewFromKubectlContext creates a test cluster from a kubectl config. +// +// If the kubectl config is not specified, the default kubectl config is used. +// If the kubectl context is not specified, the default context within the +// kubectl config is used. +func NewFromKubectlContext(ctx context.Context, kubectlContext string) ([]*testcluster.TestCluster, error) { + cluster, err := kubectl.NewCluster(kubectlContext) + if err != nil { + return nil, fmt.Errorf("cannot initialize cluster from kubectl config: %w", err) + } + clusterName := "test-cluster" // Default name. + if kubectlContext != "" { + clusterName = kubectlContext + } + return []*testcluster.TestCluster{testcluster.NewTestClusterFromClient(clusterName, cluster.Client())}, nil +} + +// verifyClusters verifies that all clusters are working. +func verifyClusters(ctx context.Context, clusters []*testcluster.TestCluster) error { + var g errgroup.Group + for _, cluster := range clusters { + c := cluster + g.Go(func() error { + return c.SanityCheck(ctx) + }) + } + return g.Wait() +} diff --git a/test/kubernetes/test_range_config.proto b/test/kubernetes/test_range_config.proto index a0ea255f9b..2f6136fa68 100644 --- a/test/kubernetes/test_range_config.proto +++ b/test/kubernetes/test_range_config.proto @@ -64,10 +64,24 @@ message TestRange { // Cluster holds the created cluster and its credential file. message Cluster { - // Created cluster proto. - google.protobuf.Any cluster = 1; + // Cluster name. + string cluster = 1; - // The setup step will create individual credential files for each created - // cluster. - string credential_file = 2; + // A kubectl config file that can be used to connect to the cluster. + string kubectl_config = 2; + + // The kubectl context name within `kubectl_config` to use to connect + // to the cluster. + // If empty, the default context will be used. + string kubectl_context = 3; + + // The GCP project ID that the cluster is in. + // Optional; only used for cluster management tasks (e.g. deletion). + // Leave empty for non-GCP clusters. + string gcp_project_id = 4; + + // The GCP location that the cluster is created in. + // Optional; only used for cluster management tasks (e.g. deletion). + // Leave empty for non-GCP clusters. + string gcp_location = 5; } diff --git a/test/kubernetes/testcluster/objects.go b/test/kubernetes/testcluster/objects.go index 9ac5513512..7349717b15 100644 --- a/test/kubernetes/testcluster/objects.go +++ b/test/kubernetes/testcluster/objects.go @@ -310,6 +310,47 @@ func (n *Namespace) WaitForResources(ctx context.Context, requests ContainerReso } } +// SanityCheck ensures that the cluster is working. +// It does so by creating sanity-check pod in the sanity namespace and ensure +// it runs successfully. +func (t *TestCluster) SanityCheck(ctx context.Context) error { + sanityNS := t.Namespace(NamespaceSanity) + resetCtx, resetCancel := context.WithTimeout(ctx, 6*time.Minute) + defer resetCancel() + defer sanityNS.Cleanup(resetCtx) + sanityCtx, sanityCancel := context.WithTimeout(resetCtx, 5*time.Minute) + defer sanityCancel() + var lastErr error + for sanityCtx.Err() == nil { + err := func() error { + if err := sanityNS.Reset(sanityCtx); err != nil { + return fmt.Errorf("cannot reset %v namespace: %w", NamespaceSanity, err) + } + defer sanityNS.Cleanup(sanityCtx) + sanityPod := sanityNS.NewAlpinePod("check", "alpine", []string{"/bin/sh", "-c", "echo", "hello"}) + sanityPod, err := t.CreatePod(sanityCtx, sanityPod) + if err != nil { + return fmt.Errorf("cannot create sanity check pod: %w", err) + } + if err := t.WaitForPodCompleted(sanityCtx, sanityPod); err != nil { + _ = t.DeletePod(resetCtx, sanityPod) + return fmt.Errorf("failed waiting for sanity check pod to complete: %w", err) + } + if err := t.DeletePod(sanityCtx, sanityPod); err != nil { + return fmt.Errorf("cannot delete sanity check pod: %w", err) + } + return nil + }() + if err == nil { + return nil + } + if sanityCtx.Err() == nil { + lastErr = err + } + } + return fmt.Errorf("cannot ensure cluster %v is working: %w", t.GetName(), lastErr) +} + // RuntimeType is a supported runtime for the test nodepool. type RuntimeType string diff --git a/test/kubernetes/testcluster/testcluster.go b/test/kubernetes/testcluster/testcluster.go index a21dee7869..80319ef339 100644 --- a/test/kubernetes/testcluster/testcluster.go +++ b/test/kubernetes/testcluster/testcluster.go @@ -25,7 +25,6 @@ import ( "time" "golang.org/x/sync/errgroup" - cspb "google.golang.org/genproto/googleapis/container/v1" "gvisor.dev/gvisor/pkg/log" "gvisor.dev/gvisor/pkg/sync" testpb "gvisor.dev/gvisor/test/kubernetes/test_range_config_go_proto" @@ -192,20 +191,23 @@ type NodePool struct { // NewTestClusterFromProto returns a new TestCluster client from a proto. func NewTestClusterFromProto(ctx context.Context, cluster *testpb.Cluster) (*TestCluster, error) { - config, err := clientcmd.BuildConfigFromFlags("" /*masterURL*/, cluster.GetCredentialFile()) + kubeCfg, err := clientcmd.LoadFromFile(cluster.GetKubectlConfig()) if err != nil { - return nil, fmt.Errorf("BuildConfigFromFlags: %w", err) + return nil, fmt.Errorf("cannot load kubectl config at %q: %w", cluster.GetKubectlConfig(), err) } - client, err := kubernetes.NewForConfig(config) + kubeContext := cluster.GetKubectlContext() + if kubeContext == "" { + kubeContext = kubeCfg.CurrentContext + } + restCfg, err := clientcmd.NewNonInteractiveClientConfig(*kubeCfg, kubeContext, nil, clientcmd.NewDefaultClientConfigLoadingRules()).ClientConfig() if err != nil { - return nil, fmt.Errorf("kubernetes.NewForConfig: %w", err) + return nil, fmt.Errorf("kubectl config: %w", err) } - var clusterPB cspb.Cluster - if err := cluster.GetCluster().UnmarshalTo(&clusterPB); err != nil { - return nil, fmt.Errorf("cannot unmarshal cluster: %w", err) + client, err := kubernetes.NewForConfig(restCfg) + if err != nil { + return nil, fmt.Errorf("kubernetes.NewForConfig: %w", err) } - clusterName := clusterPB.GetName() - return NewTestClusterFromClient(clusterName, client), nil + return NewTestClusterFromClient(cluster.GetCluster(), client), nil } // NewTestClusterFromClient returns a new TestCluster client with a given client. diff --git a/test/kubernetes/tests/BUILD b/test/kubernetes/tests/BUILD index 88298194ab..8f3937816a 100644 --- a/test/kubernetes/tests/BUILD +++ b/test/kubernetes/tests/BUILD @@ -27,5 +27,5 @@ go_test( "noguitar", "notap", ], - deps = ["//test/kubernetes/k8sctx/autok8sctx"], + deps = ["//test/kubernetes/k8sctx/kubectlctx"], ) diff --git a/test/kubernetes/tests/hello_test.go b/test/kubernetes/tests/hello_test.go index eb71d238c6..5910500cb2 100644 --- a/test/kubernetes/tests/hello_test.go +++ b/test/kubernetes/tests/hello_test.go @@ -18,13 +18,13 @@ import ( "context" "testing" - "gvisor.dev/gvisor/test/kubernetes/k8sctx/autok8sctx" + "gvisor.dev/gvisor/test/kubernetes/k8sctx/kubectlctx" ) // TestHello tests that a trivial alpine container runs correctly. func TestHello(t *testing.T) { ctx := context.Background() - k8sCtx, err := autok8sctx.New(ctx) + k8sCtx, err := kubectlctx.New(ctx) if err != nil { t.Fatalf("Failed to get kubernetes context: %v", err) }