Skip to content

Commit

Permalink
Merge pull request #8844 from zalando-incubator/dev-to-alpha
Browse files Browse the repository at this point in the history
dev to alpha
  • Loading branch information
katyanna authored Jan 30, 2025
2 parents 48ebc0d + c87d699 commit e70e157
Show file tree
Hide file tree
Showing 12 changed files with 226 additions and 9 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ spec:
hostNetwork: true
containers:
- name: kube-aws-iam-controller
image: container-registry.zalando.net/teapot/kube-aws-iam-controller:v0.3.0-49-g8369d21
image: container-registry.zalando.net/teapot/kube-aws-iam-controller:v0.3.0-52-g5502e4d
env:
- name: AWS_DEFAULT_REGION
value: "{{.Cluster.Region}}"
Expand Down
6 changes: 3 additions & 3 deletions cluster/manifests/03-ebs-csi/controller.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ spec:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
- name: csi-provisioner
image: container-registry.zalando.net/teapot/external-provisioner:v5.1.0-eks-1-31-10-master-25
image: container-registry.zalando.net/teapot/external-provisioner:v5.1.0-eks-1-31-10-master-26
args:
- --csi-address=$(ADDRESS)
- --v=2
Expand All @@ -107,7 +107,7 @@ spec:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
- name: csi-attacher
image: container-registry.zalando.net/teapot/external-attacher:v4.7.0-eks-1-31-10-master-25
image: container-registry.zalando.net/teapot/external-attacher:v4.7.0-eks-1-31-10-master-26
args:
- --csi-address=$(ADDRESS)
- --v=2
Expand All @@ -129,7 +129,7 @@ spec:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
- name: csi-resizer
image: container-registry.zalando.net/teapot/external-resizer:v1.12.0-eks-1-31-10-master-25
image: container-registry.zalando.net/teapot/external-resizer:v1.12.0-eks-1-31-10-master-26
args:
- --csi-address=$(ADDRESS)
- --v=2
Expand Down
4 changes: 2 additions & 2 deletions cluster/manifests/03-ebs-csi/node.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ spec:
runAsUser: 0
containers:
- name: ebs-plugin
image: container-registry.zalando.net/teapot/aws-ebs-csi-driver:v1.38.1-master-24
image: container-registry.zalando.net/teapot/aws-ebs-csi-driver:v1.38.1-master-26
args:
- node
- --endpoint=$(CSI_ENDPOINT)
Expand Down Expand Up @@ -77,7 +77,7 @@ spec:
privileged: true
readOnlyRootFilesystem: true
- name: node-driver-registrar
image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-25
image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-26
args:
- --csi-address=$(ADDRESS)
- --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
Expand Down
33 changes: 33 additions & 0 deletions cluster/manifests/kube-metrics-adapter/01-rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -169,3 +169,36 @@ subjects:
- kind: ServiceAccount
name: custom-metrics-apiserver
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kube-metrics-adapter-scaling-schedule-scaler
labels:
application: kubernetes
component: kube-metrics-adapter
rules:
- apiGroups:
- "apps"
resources:
- deployments/scale
- statefulsets/scale
verbs:
- get
- update
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kube-metrics-adapter-scaling-schedule-scaler
labels:
application: kubernetes
component: kube-metrics-adapter
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kube-metrics-adapter-scaling-schedule-scaler
subjects:
- kind: ServiceAccount
name: custom-metrics-apiserver
namespace: kube-system
2 changes: 1 addition & 1 deletion cluster/manifests/kube-metrics-adapter/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ spec:
serviceAccountName: custom-metrics-apiserver
containers:
- name: kube-metrics-adapter
image: container-registry.zalando.net/teapot/kube-metrics-adapter:kube-metrics-adapter-0.2.3-46-ge580c5b
image: container-registry.zalando.net/teapot/kube-metrics-adapter:kube-metrics-adapter-0.2.3-49-gd416441
env:
- name: AWS_REGION
value: {{ .Cluster.Region }}
Expand Down
2 changes: 1 addition & 1 deletion cluster/manifests/s3-csi-driver/daemonset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ spec:
valueFrom:
fieldRef:
fieldPath: spec.nodeName
image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-24
image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-26
imagePullPolicy: IfNotPresent
name: node-driver-registrar
resources:
Expand Down
File renamed without changes.
22 changes: 22 additions & 0 deletions delivery.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ pipeline:
value: "teapot-kubernetes-e2e-results"
- name: ETCD_ENDPOINTS
value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479"
- name: SKIPPER_OPA_ENABLED
value: "true"
- name: HOSTED_ZONE
valueFrom:
configMapKeyRef:
Expand Down Expand Up @@ -155,6 +157,26 @@ pipeline:
secretKeyRef:
name: kubernetes-e2e-config-secret
key: "OKTA_AUTH_ISSUER_URL"
- name: STYRA_TOKEN
valueFrom:
secretKeyRef:
name: kubernetes-e2e-config-secret
key: "STYRA_TOKEN"
- name: SKIPPER_OPA_BUCKET_ARN
valueFrom:
secretKeyRef:
name: kubernetes-e2e-config-secret
key: "SKIPPER_OPA_BUCKET_ARN"
- name: SKIPPER_OPA_OBSERVABILITY_URL
valueFrom:
secretKeyRef:
name: kubernetes-e2e-config-secret
key: "SKIPPER_OPA_OBSERVABILITY_URL"
- name: SKIPPER_OPA_BUNDLES_URL
valueFrom:
secretKeyRef:
name: kubernetes-e2e-config-secret
key: "SKIPPER_OPA_BUNDLES_URL"
- name: CLUSTER_ADMIN_TOKEN
valueFrom:
secretKeyRef:
Expand Down
153 changes: 153 additions & 0 deletions test/e2e/aaas.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
package e2e

import (
"context"
"fmt"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"

netv1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/test/e2e/framework"
"k8s.io/kubernetes/test/e2e/framework/ingress"
admissionapi "k8s.io/pod-security-admission/api"
"net/http"
"time"
)

var _ = Describe("Ingress tests for OPA filters", func() {
f := framework.NewDefaultFramework("skipper-ingress-with-opa")
f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline

var (
cs kubernetes.Interface
jig *ingress.TestJig
ingressCreate *netv1.Ingress
ns string
hostName string
port int
serviceName = "aaas-infrastructure-tests"
opaPolicyName = "aaas-infrastructure-tests"
)

BeforeEach(func() {
jig = ingress.NewIngressTestJig(f.ClientSet)
cs = f.ClientSet
ns = f.Namespace.Name
hostName = fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), E2EHostedZone())
port = 8080

ingressCreate = createIngressWithInfo(serviceName, hostName, ns, port, cs, jig)
})

It("opaAuthorizeRequest filter should pass through authorized requests [Ingress] [Opa]", func() {
// Authorization done with the rule
// https://github.bus.zalan.do/corporate-iam/aaas-infrastructure-tests-policies/blob/main/bundle/policy/ingress/rules.rego

authorizationEnforcedPath := "/auth"
path := "/"
ingressRoute := fmt.Sprintf(`authorize: PathRegexp("%s") -> modPath("%s", "%s") -> opaAuthorizeRequest("%s") -> inlineContent("Got response") -> <shunt>;`, authorizationEnforcedPath, authorizationEnforcedPath, path, opaPolicyName)
ingressUpdate := updateIngressAndWait(serviceName, hostName, path, ingressRoute, port, ingressCreate, cs)

By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
rt, quit := createHTTPRoundTripper()
defer func() {
quit <- struct{}{}
}()

url := "https://" + hostName + authorizationEnforcedPath
req, err := http.NewRequest("GET", url, nil)
resp, err := getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden)
framework.ExpectNoError(err)
Expect(resp.StatusCode).To(Equal(http.StatusForbidden))

By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
req.Header.Set("Authorization", "Basic valid_token") // Authorized request
resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK)
framework.ExpectNoError(err)
Expect(resp.StatusCode).To(Equal(http.StatusOK))
s, err := getBody(resp)
framework.ExpectNoError(err)
Expect(s).To(Equal("Got response"))
})

It("opaServeResponse filter should return body [Ingress] [Opa]", func() {
// Authorization done with the rule
// https://github.bus.zalan.do/corporate-iam/aaas-infrastructure-tests-policies/blob/main/bundle/policy/ingress/permissions.rego

serveResponsePath := "/permissions"
expectedPermissionsContent := "\"permissions\":{\"permission1\":{},\"permission2\":{}}"
path := serveResponsePath
ingressRoute := fmt.Sprintf(`authorize: PathRegexp("%s") -> opaServeResponse("%s") -> <shunt>;`, serveResponsePath, opaPolicyName)
ingressUpdate := updateIngressAndWait(serviceName, hostName, path, ingressRoute, port, ingressCreate, cs)

By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 403 with opaServeResponse %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
rt, quit := createHTTPRoundTripper()
defer func() {
quit <- struct{}{}
}()

url := "https://" + hostName + serveResponsePath
req, err := http.NewRequest("GET", url, nil)
resp, err := getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden)
framework.ExpectNoError(err)
Expect(resp.StatusCode).To(Equal(http.StatusForbidden))

By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 200 with opaServeResponse %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
req.Header.Set("Authorization", "Basic permissions_token") // Authorized request
resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK)
framework.ExpectNoError(err)
Expect(resp.StatusCode).To(Equal(http.StatusOK))
s, err := getBody(resp)
framework.ExpectNoError(err)
Expect(s).To(ContainSubstring(expectedPermissionsContent))
})

})

func createIngressWithInfo(serviceName, hostName, ns string, port int, cs kubernetes.Interface, jig *ingress.TestJig) *netv1.Ingress {
labels := map[string]string{"app": serviceName}
waitTime := 10 * time.Minute

// Create initial ingress
ing := createIngress(serviceName, hostName, ns, "/", netv1.PathTypeImplementationSpecific, labels, nil, port)
var err error
ingressCreate, err := cs.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{})
framework.ExpectNoError(err)

// Wait for ingress address
addr, err := jig.WaitForIngressAddress(context.TODO(), cs, ns, ingressCreate.Name, waitTime)
framework.ExpectNoError(err)

// Ensure ingress exists
_, err = cs.NetworkingV1().Ingresses(ns).Get(context.TODO(), ing.Name, metav1.GetOptions{ResourceVersion: "0"})
framework.ExpectNoError(err)

By("Waiting for LB to create endpoint and skipper route")
err = waitForResponse(addr, "https", waitTime, isNotFound, true)
framework.ExpectNoError(err)

return ingressCreate
}

func updateIngressAndWait(serviceName, hostName, path, ingressRoute string, port int, ingressCreate *netv1.Ingress, cs kubernetes.Interface) *netv1.Ingress {
updatedIng := updateIngress(ingressCreate.ObjectMeta.Name,
ingressCreate.ObjectMeta.Namespace,
hostName,
serviceName,
path,
netv1.PathTypeImplementationSpecific,
ingressCreate.ObjectMeta.Labels,
map[string]string{
"zalando.org/skipper-routes": ingressRoute,
},
port,
)
ingressUpdate, err := cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{})
framework.ExpectNoError(err)
time.Sleep(2 * time.Minute) // wait for routing change propagation

return ingressUpdate
}
4 changes: 4 additions & 0 deletions test/e2e/apply/secret.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,3 +19,7 @@ data:
ETCD_SCALYR_KEY: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwETRzvm1hGplyUn23FEXUVtAAAAnjCBmwYJKoZIhvcNAQcGoIGNMIGKAgEAMIGEBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOfPJJJy60sDkZEIHgIBEIBXiANNciEqpcuZ3hFPCt6NkFtk0WBTSasDQHHbyuR8O+n5iM9k8/nUTLUrFlhba8blArq/ALE8vuKNdlS17q6PxGlvwJFFXQn/McohMpdyfnfQYKW8MPCu"
OKTA_AUTH_ISSUER_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGmCMhSN2Er1sw2ofYnI44EAAAApDCBoQYJKoZIhvcNAQcGoIGTMIGQAgEAMIGKBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDO2IC+r/zcUzXoQEHAIBEIBdrFchwu9i7LpMbyDbslu/lBxvfyh+nCGK33jtcxT3RdxuTXWuSJhkX+gU4cgFXAI5LLnXh4M20jHUEEPU78MJWR47HLTPGPJcKQj5fOpPqpD3duuKIrZDRm5ba6AN"
SESSION_MANAGER_DESTINATION_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwF7fOZ9i6BDvWdNEddR7LZOAAAArjCBqwYJKoZIhvcNAQcGoIGdMIGaAgEAMIGUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBJwU/Zns+mzOBgczQIBEIBn/86xpnVO2Apr5nG3waPEAGCFYDWdOXcaS7pFKdNIhpXaADtODQtEd874HcE0W2I3bjKr3d3ghJFdN8r0BZiSmTbgc0fn+5ZiBTyGBfzWP4BCzxjRMvURl/7MX8ygwL78hpSxyRypAQ=="
STYRA_TOKEN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGba1bnMYs8qNGHMd7a66H8AAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDE/go6xBMbsPDNgTsQIBEIB7E+VUre5YvyOJ8Apbi1SXMqNsr1rMp38is8iM5nP3LXHknTKIBZMgTeRilreIGQ25+UjWjBrTRUWmmD5jq6oN/d8t+0AEcLOpGZUplZSV+za31/3pAOPCFSqhUgzlUiI6LWW1kNKtmWwNh9aDEu4geiOAJSxJTAot6H77"
SKIPPER_OPA_BUCKET_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGZdCVDLsCdProfzvZU7UAwAAAAlzCBlAYJKoZIhvcNAQcGoIGGMIGDAgEAMH4GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMehOf7Uu444SWS6kbAgEQgFFPMaa0flwHLpxrkYjJMK4jXc0q4kX+KGrB5GFjKuUgOUPmQ+ME/aQduxwl2+xUilrKP50/NLXgMNHjeeHuZfoyiSgpGFBM4z8L0N6ggf2uE5U="
SKIPPER_OPA_OBSERVABILITY_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwHl773AuNEvIpzaM6ycpDNSAAAAqzCBqAYJKoZIhvcNAQcGoIGaMIGXAgEAMIGRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGld6jpQ38gOzVcn0gIBEIBkTHbv3adeEfRntVTUQyyQkIhUnc0QXKtmtJEdvBoRzWiJIBKQUQuM1VBV0re3HkO8HSY59nkwyHEncBMkHJoI9rC2LJuWU20oCjPw9lbweih+6Sxo+nqkDrQd+mHp+uA9Om3KqA=="
SKIPPER_OPA_BUNDLES_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwFnhaIRP4+3Y69xp1ycTI7qAAAAsTCBrgYJKoZIhvcNAQcGoIGgMIGdAgEAMIGXBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDF9gAl70l2g2kwfnJgIBEIBqP/DgIhIu5x5XNR1Ubqinz6r4ttQoHty8nXd6mxie2r6NxHskNOqkiSactUKhNIhboNlNsO4p4rKEkhglTeFZlEQvgEYNioWPw39xqICnUDPVr+Kp0Yrs/bzPLPV9wOlB917UiT7WJNybPg=="
5 changes: 5 additions & 0 deletions test/e2e/cluster_config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,11 @@ clusters:
karpenter_pools_enabled: "true"
okta_auth_client_id: "kubernetes.cluster.teapot-e2e"
teapot_admission_controller_validate_pod_images_soft_fail_namespaces: "^kube-system$"
skipper_open_policy_agent_enabled: "${SKIPPER_OPA_ENABLED}"
skipper_open_policy_agent_styra_token: "${STYRA_TOKEN}"
skipper_open_policy_agent_bucket_arn: "${SKIPPER_OPA_BUCKET_ARN}"
skipper_open_policy_agent_observability_url: "${SKIPPER_OPA_OBSERVABILITY_URL}"
skipper_open_policy_agent_bundles_url: "${SKIPPER_OPA_BUNDLES_URL}"
criticality_level: 1
environment: e2e
id: ${CLUSTER_ID}
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/run_e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ if [ "$e2e" = true ]; then

mkdir -p junit_reports
ginkgo -procs=25 -flake-attempts=2 \
-focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \
-focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\]|\[Opa\])" \
-skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \
"e2e.test" -- \
-delete-namespace-on-failure=false \
Expand Down

0 comments on commit e70e157

Please sign in to comment.