Skip to content

Commit 4dafb7d

Browse files
mythimythi
committed
WIP: e2e: add initial SGX EPC cgroups tests
Signed-off-by: Mikko Ylinen <[email protected]>
1 parent 2e3901c commit 4dafb7d

File tree

3 files changed

+119
-3
lines changed

3 files changed

+119
-3
lines changed

‎Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ e2e-dlb:
172172
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:dlb.*$(ADDITIONAL_FOCUS_REGEX)" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
173173

174174
e2e-spr:
175-
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:(iaa|dsa)|Device:qat.*Mode:dpdk.*Resource:(cy|dc).*" -ginkgo.focus "Device:sgx.*|(SGX Admission)" -ginkgo.focus "Device:gpu.*Resource:i915" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
175+
@$(GO) test -v ./test/e2e/... -ginkgo.v -ginkgo.show-node-events -ginkgo.focus "Device:(iaa|dsa)|Device:qat.*Mode:dpdk.*Resource:(cy|dc).*" -ginkgo.focus "Device:sgx.*|(SGX Admission)" -ginkgo.focus "Device:gpu.*Resource:i915" -ginkgo.skip "App:sgx-epc-cgroup" $(GENERATED_SKIP_OPT) -delete-namespace-on-failure=false
176176

177177
pre-pull:
178178
ifeq ($(TAG),devel)

‎scripts/set-version.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ if [ $# != 1 ] || [ "$1" = "?" ] || [ "$1" = "--help" ]; then
1515
exit 1
1616
fi
1717

18-
files=$(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml demo/*openssl*.yaml demo/dlb-libdlb*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile test/e2e/*/*.go)
18+
files=$(git grep -l '^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/stress-ng-gramine:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion:\|tag:' Makefile deployments demo/*accel-config*.yaml demo/*fpga*.yaml demo/*openssl*.yaml demo/dlb-libdlb*.yaml pkg/controllers/*/*_test.go build/docker/*.Dockerfile test/e2e/*/*.go)
1919

2020
for file in $files; do
21-
sed -i -e "s;\(^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
21+
sed -i -e "s;\(^TAG?*=\|intel/accel-config-demo:\|intel/crypto-perf:\|intel/opae-nlb-demo:\|intel/openssl-qat-engine:\|intel/dlb-libdlb-demo:\|intel/stress-ng-gramine:\|intel/sgx-sdk-demo:\|intel/intel-[^ ]*:\|version=\|appVersion: [^ ]\|tag: [^ ]\)[^ \"]*;\1$1;g" "$file";
2222
done

‎test/e2e/sgx/sgx.go

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ package sgx
1616

1717
import (
1818
"context"
19+
"fmt"
1920
"path/filepath"
2021
"time"
2122

@@ -28,6 +29,7 @@ import (
2829
"k8s.io/apimachinery/pkg/labels"
2930
"k8s.io/kubernetes/test/e2e/framework"
3031
e2edebug "k8s.io/kubernetes/test/e2e/framework/debug"
32+
e2ejob "k8s.io/kubernetes/test/e2e/framework/job"
3133
e2ekubectl "k8s.io/kubernetes/test/e2e/framework/kubectl"
3234
e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
3335
admissionapi "k8s.io/pod-security-admission/api"
@@ -38,6 +40,10 @@ const (
3840
timeout = time.Second * 120
3941
kustomizationWebhook = "deployments/sgx_admissionwebhook/overlays/default-with-certmanager/kustomization.yaml"
4042
kustomizationPlugin = "deployments/sgx_plugin/base/kustomization.yaml"
43+
// TODO: move to epc-cgroups overlay once available.
44+
// kustomizationPlugin = "deployments/sgx_plugin/overlays/epc-cgroups/kustomization.yaml".
45+
stressNGImage = "intel/stress-ng-gramine:devel"
46+
stressNGEnclaveSize = 402653184
4147
)
4248

4349
func init() {
@@ -80,6 +86,9 @@ func describe() {
8086
})
8187

8288
ginkgo.Context("When SGX resources are available", func() {
89+
var nodeWithEPC string
90+
var epcCapacity int64
91+
8392
ginkgo.BeforeEach(func(ctx context.Context) {
8493
ginkgo.By("checking if the resource is allocatable")
8594
if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "sgx.intel.com/epc", 150*time.Second, utils.WaitForPositiveResource); err != nil {
@@ -91,6 +100,20 @@ func describe() {
91100
if err := utils.WaitForNodesWithResource(ctx, f.ClientSet, "sgx.intel.com/provision", 30*time.Second, utils.WaitForPositiveResource); err != nil {
92101
framework.Failf("unable to wait for nodes to have positive allocatable provision resource: %v", err)
93102
}
103+
104+
nodelist, err := f.ClientSet.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
105+
if err != nil {
106+
framework.Failf("failed to list Nodes: %v", err)
107+
}
108+
109+
// we have at least one node with sgx.intel.com/epc capacity
110+
for _, item := range nodelist.Items {
111+
if q, ok := item.Status.Allocatable["sgx.intel.com/epc"]; ok && q.Value() > 0 {
112+
epcCapacity = q.Value()
113+
nodeWithEPC = item.Name
114+
break
115+
}
116+
}
94117
})
95118

96119
ginkgo.It("deploys a sgx-sdk-demo pod requesting SGX enclave resources [App:sgx-sdk-demo]", func(ctx context.Context) {
@@ -120,6 +143,99 @@ func describe() {
120143
gomega.Expect(err).To(gomega.BeNil(), utils.GetPodLogs(ctx, f, pod.ObjectMeta.Name, "testcontainer"))
121144
})
122145

146+
ginkgo.It("deploys simultaneous SGX EPC stressor jobs with equal EPC limits but no memory limits [App:sgx-epc-cgroup]", func(ctx context.Context) {
147+
parallelism := int32(epcCapacity/stressNGEnclaveSize) + 1
148+
completions := int32(epcCapacity/stressNGEnclaveSize) + 1
149+
quantity := resource.NewQuantity(stressNGEnclaveSize/2, resource.BinarySI)
150+
151+
testArgs := []string{
152+
"stress-ng",
153+
"--verbose",
154+
"--vm",
155+
"1",
156+
"--vm-bytes",
157+
"20%",
158+
"--page-in",
159+
"-t",
160+
"30",
161+
}
162+
job := e2ejob.NewTestJobOnNode("success", fmt.Sprintf("sgx-epc-stressjob-npods-%d", parallelism), v1.RestartPolicyNever, parallelism, completions, nil, 0, nodeWithEPC)
163+
164+
job.Spec.Template.Spec.Containers[0].Image = stressNGImage
165+
job.Spec.Template.Spec.Containers[0].Args = testArgs
166+
job.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{
167+
Requests: v1.ResourceList{"sgx.intel.com/epc": *quantity},
168+
Limits: v1.ResourceList{"sgx.intel.com/epc": *quantity},
169+
}
170+
171+
job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
172+
framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)
173+
174+
err = e2ejob.WaitForJobComplete(ctx, f.ClientSet, f.Namespace.Name, job.Name, nil, completions)
175+
framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
176+
})
177+
178+
ginkgo.It("deploys one SGX EPC stressor job with the EPC limit set to enclave size and the memory limit set very low [App:sgx-epc-cgroup]", func(ctx context.Context) {
179+
quantity := resource.NewQuantity(stressNGEnclaveSize, resource.BinarySI)
180+
181+
testArgs := []string{
182+
"stress-ng",
183+
"--verbose",
184+
"--vm",
185+
"1",
186+
"--vm-bytes",
187+
"20%",
188+
"--page-in",
189+
"-t",
190+
"30",
191+
}
192+
job := e2ejob.NewTestJobOnNode("success", "sgx-epc-stressjob-lowmemlimit", v1.RestartPolicyNever, 1, 1, nil, 0, nodeWithEPC)
193+
194+
job.Spec.Template.Spec.Containers[0].Image = stressNGImage
195+
job.Spec.Template.Spec.Containers[0].Args = testArgs
196+
job.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{
197+
Requests: v1.ResourceList{"sgx.intel.com/epc": *quantity},
198+
Limits: v1.ResourceList{"sgx.intel.com/epc": *quantity,
199+
v1.ResourceMemory: resource.MustParse("42Mi")},
200+
}
201+
202+
job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
203+
framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)
204+
205+
err = e2ejob.WaitForJobComplete(ctx, f.ClientSet, f.Namespace.Name, job.Name, nil, 1)
206+
framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
207+
})
208+
209+
ginkgo.It("deploys one SGX EPC stressor job with EDMM that ramps EPC allocations and memory limit set to kill once enough EPC pages are reclaimed [App:sgx-epc-cgroup]", func(ctx context.Context) {
210+
quantity := resource.NewQuantity(epcCapacity/10, resource.BinarySI)
211+
212+
testArgs := []string{
213+
"stress-ng-edmm",
214+
"--verbose",
215+
"--bigheap",
216+
"1",
217+
"--bigheap-growth",
218+
"10m",
219+
"--page-in",
220+
"-t",
221+
"300",
222+
}
223+
job := e2ejob.NewTestJobOnNode("success", "sgx-epc-stressjob-oom", v1.RestartPolicyNever, 1, 1, nil, 0, nodeWithEPC)
224+
225+
job.Spec.Template.Spec.Containers[0].Image = stressNGImage
226+
job.Spec.Template.Spec.Containers[0].Args = testArgs
227+
job.Spec.Template.Spec.Containers[0].Resources = v1.ResourceRequirements{
228+
Requests: v1.ResourceList{"sgx.intel.com/epc": *quantity},
229+
Limits: v1.ResourceList{"sgx.intel.com/epc": *quantity,
230+
v1.ResourceMemory: *quantity},
231+
}
232+
233+
job, err := e2ejob.CreateJob(ctx, f.ClientSet, f.Namespace.Name, job)
234+
framework.ExpectNoError(err, "failed to create job in namespace: %s", f.Namespace.Name)
235+
err = e2ejob.WaitForJobFailed(f.ClientSet, f.Namespace.Name, job.Name)
236+
framework.ExpectNoError(err, "failed to ensure job completion in namespace: %s", f.Namespace.Name)
237+
})
238+
123239
ginkgo.When("there is no app to run [App:noapp]", func() {
124240
ginkgo.It("does nothing", func() {})
125241
})

0 commit comments

Comments
 (0)