dev to alpha

cluster/manifests/02-kube-aws-iam-controller/deployment.yaml

@@ -27,7 +27,7 @@ spec:
       hostNetwork: true
       containers:
       - name: kube-aws-iam-controller
-        image: container-registry.zalando.net/teapot/kube-aws-iam-controller:v0.3.0-49-g8369d21
+        image: container-registry.zalando.net/teapot/kube-aws-iam-controller:v0.3.0-52-g5502e4d
         env:
         - name: AWS_DEFAULT_REGION
           value: "{{.Cluster.Region}}"

cluster/manifests/03-ebs-csi/controller.yaml

@@ -82,7 +82,7 @@ spec:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
         - name: csi-provisioner
-          image: container-registry.zalando.net/teapot/external-provisioner:v5.1.0-eks-1-31-10-master-25
+          image: container-registry.zalando.net/teapot/external-provisioner:v5.1.0-eks-1-31-10-master-26
           args:
             - --csi-address=$(ADDRESS)
             - --v=2
@@ -107,7 +107,7 @@ spec:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
         - name: csi-attacher
-          image: container-registry.zalando.net/teapot/external-attacher:v4.7.0-eks-1-31-10-master-25
+          image: container-registry.zalando.net/teapot/external-attacher:v4.7.0-eks-1-31-10-master-26
           args:
             - --csi-address=$(ADDRESS)
             - --v=2
@@ -129,7 +129,7 @@ spec:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
         - name: csi-resizer
-          image: container-registry.zalando.net/teapot/external-resizer:v1.12.0-eks-1-31-10-master-25
+          image: container-registry.zalando.net/teapot/external-resizer:v1.12.0-eks-1-31-10-master-26
           args:
             - --csi-address=$(ADDRESS)
             - --v=2

cluster/manifests/03-ebs-csi/node.yaml

@@ -34,7 +34,7 @@ spec:
         runAsUser: 0
       containers:
         - name: ebs-plugin
-          image: container-registry.zalando.net/teapot/aws-ebs-csi-driver:v1.38.1-master-24
+          image: container-registry.zalando.net/teapot/aws-ebs-csi-driver:v1.38.1-master-26
           args:
             - node
             - --endpoint=$(CSI_ENDPOINT)
@@ -77,7 +77,7 @@ spec:
             privileged: true
             readOnlyRootFilesystem: true
         - name: node-driver-registrar
-          image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-25
+          image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-26
           args:
             - --csi-address=$(ADDRESS)
             - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)

cluster/manifests/kube-metrics-adapter/01-rbac.yaml

@@ -169,3 +169,36 @@ subjects:
 - kind: ServiceAccount
   name: custom-metrics-apiserver
   namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kube-metrics-adapter-scaling-schedule-scaler
+  labels:
+    application: kubernetes
+    component: kube-metrics-adapter
+rules:
+- apiGroups:
+  - "apps"
+  resources:
+  - deployments/scale
+  - statefulsets/scale
+  verbs:
+  - get
+  - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kube-metrics-adapter-scaling-schedule-scaler
+  labels:
+    application: kubernetes
+    component: kube-metrics-adapter
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-metrics-adapter-scaling-schedule-scaler
+subjects:
+- kind: ServiceAccount
+  name: custom-metrics-apiserver
+  namespace: kube-system

cluster/manifests/kube-metrics-adapter/deployment.yaml

@@ -27,7 +27,7 @@ spec:
       serviceAccountName: custom-metrics-apiserver
       containers:
       - name: kube-metrics-adapter
-        image: container-registry.zalando.net/teapot/kube-metrics-adapter:kube-metrics-adapter-0.2.3-46-ge580c5b
+        image: container-registry.zalando.net/teapot/kube-metrics-adapter:kube-metrics-adapter-0.2.3-49-gd416441
         env:
         - name: AWS_REGION
           value: {{ .Cluster.Region }}

cluster/manifests/s3-csi-driver/daemonset.yaml

@@ -95,7 +95,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: spec.nodeName
-        image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-24
+        image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-26
         imagePullPolicy: IfNotPresent
         name: node-driver-registrar
         resources:

delivery.yaml

@@ -80,6 +80,8 @@ pipeline:
             value: "teapot-kubernetes-e2e-results"
           - name: ETCD_ENDPOINTS
             value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479"
+          - name: SKIPPER_OPA_ENABLED
+            value: "true"
           - name: HOSTED_ZONE
             valueFrom:
               configMapKeyRef:
@@ -155,6 +157,26 @@ pipeline:
               secretKeyRef:
                 name: kubernetes-e2e-config-secret
                 key: "OKTA_AUTH_ISSUER_URL"
+          - name: STYRA_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: kubernetes-e2e-config-secret
+                key: "STYRA_TOKEN"
+          - name: SKIPPER_OPA_BUCKET_ARN
+            valueFrom:
+              secretKeyRef:
+                name: kubernetes-e2e-config-secret
+                key: "SKIPPER_OPA_BUCKET_ARN"
+          - name: SKIPPER_OPA_OBSERVABILITY_URL
+            valueFrom:
+              secretKeyRef:
+                name: kubernetes-e2e-config-secret
+                key: "SKIPPER_OPA_OBSERVABILITY_URL"
+          - name: SKIPPER_OPA_BUNDLES_URL
+            valueFrom:
+              secretKeyRef:
+                name: kubernetes-e2e-config-secret
+                key: "SKIPPER_OPA_BUNDLES_URL"
           - name: CLUSTER_ADMIN_TOKEN
             valueFrom:
               secretKeyRef:

test/e2e/aaas.go

@@ -0,0 +1,153 @@
+package e2e
+
+import (
+	"context"
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	netv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/ingress"
+	admissionapi "k8s.io/pod-security-admission/api"
+	"net/http"
+	"time"
+)
+
+var _ = Describe("Ingress tests for OPA filters", func() {
+	f := framework.NewDefaultFramework("skipper-ingress-with-opa")
+	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
+
+	var (
+		cs            kubernetes.Interface
+		jig           *ingress.TestJig
+		ingressCreate *netv1.Ingress
+		ns            string
+		hostName      string
+		port          int
+		serviceName   = "aaas-infrastructure-tests"
+		opaPolicyName = "aaas-infrastructure-tests"
+	)
+
+	BeforeEach(func() {
+		jig = ingress.NewIngressTestJig(f.ClientSet)
+		cs = f.ClientSet
+		ns = f.Namespace.Name
+		hostName = fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), E2EHostedZone())
+		port = 8080
+
+		ingressCreate = createIngressWithInfo(serviceName, hostName, ns, port, cs, jig)
+	})
+
+	It("opaAuthorizeRequest filter should pass through authorized requests [Ingress] [Opa]", func() {
+		// Authorization done with the rule
+		// https://github.bus.zalan.do/corporate-iam/aaas-infrastructure-tests-policies/blob/main/bundle/policy/ingress/rules.rego
+
+		authorizationEnforcedPath := "/auth"
+		path := "/"
+		ingressRoute := fmt.Sprintf(`authorize: PathRegexp("%s") -> modPath("%s", "%s") -> opaAuthorizeRequest("%s") -> inlineContent("Got response") -> <shunt>;`, authorizationEnforcedPath, authorizationEnforcedPath, path, opaPolicyName)
+		ingressUpdate := updateIngressAndWait(serviceName, hostName, path, ingressRoute, port, ingressCreate, cs)
+
+		By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
+		rt, quit := createHTTPRoundTripper()
+		defer func() {
+			quit <- struct{}{}
+		}()
+
+		url := "https://" + hostName + authorizationEnforcedPath
+		req, err := http.NewRequest("GET", url, nil)
+		resp, err := getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden)
+		framework.ExpectNoError(err)
+		Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
+
+		By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
+		req.Header.Set("Authorization", "Basic valid_token") // Authorized request
+		resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK)
+		framework.ExpectNoError(err)
+		Expect(resp.StatusCode).To(Equal(http.StatusOK))
+		s, err := getBody(resp)
+		framework.ExpectNoError(err)
+		Expect(s).To(Equal("Got response"))
+	})
+
+	It("opaServeResponse filter should return body [Ingress] [Opa]", func() {
+		// Authorization done with the rule
+		// https://github.bus.zalan.do/corporate-iam/aaas-infrastructure-tests-policies/blob/main/bundle/policy/ingress/permissions.rego
+
+		serveResponsePath := "/permissions"
+		expectedPermissionsContent := "\"permissions\":{\"permission1\":{},\"permission2\":{}}"
+		path := serveResponsePath
+		ingressRoute := fmt.Sprintf(`authorize: PathRegexp("%s") -> opaServeResponse("%s") -> <shunt>;`, serveResponsePath, opaPolicyName)
+		ingressUpdate := updateIngressAndWait(serviceName, hostName, path, ingressRoute, port, ingressCreate, cs)
+
+		By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 403 with opaServeResponse %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
+		rt, quit := createHTTPRoundTripper()
+		defer func() {
+			quit <- struct{}{}
+		}()
+
+		url := "https://" + hostName + serveResponsePath
+		req, err := http.NewRequest("GET", url, nil)
+		resp, err := getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden)
+		framework.ExpectNoError(err)
+		Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
+
+		By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 200 with opaServeResponse %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
+		req.Header.Set("Authorization", "Basic permissions_token") // Authorized request
+		resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK)
+		framework.ExpectNoError(err)
+		Expect(resp.StatusCode).To(Equal(http.StatusOK))
+		s, err := getBody(resp)
+		framework.ExpectNoError(err)
+		Expect(s).To(ContainSubstring(expectedPermissionsContent))
+	})
+
+})
+
+func createIngressWithInfo(serviceName, hostName, ns string, port int, cs kubernetes.Interface, jig *ingress.TestJig) *netv1.Ingress {
+	labels := map[string]string{"app": serviceName}
+	waitTime := 10 * time.Minute
+
+	// Create initial ingress
+	ing := createIngress(serviceName, hostName, ns, "/", netv1.PathTypeImplementationSpecific, labels, nil, port)
+	var err error
+	ingressCreate, err := cs.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{})
+	framework.ExpectNoError(err)
+
+	// Wait for ingress address
+	addr, err := jig.WaitForIngressAddress(context.TODO(), cs, ns, ingressCreate.Name, waitTime)
+	framework.ExpectNoError(err)
+
+	// Ensure ingress exists
+	_, err = cs.NetworkingV1().Ingresses(ns).Get(context.TODO(), ing.Name, metav1.GetOptions{ResourceVersion: "0"})
+	framework.ExpectNoError(err)
+
+	By("Waiting for LB to create endpoint and skipper route")
+	err = waitForResponse(addr, "https", waitTime, isNotFound, true)
+	framework.ExpectNoError(err)
+
+	return ingressCreate
+}
+
+func updateIngressAndWait(serviceName, hostName, path, ingressRoute string, port int, ingressCreate *netv1.Ingress, cs kubernetes.Interface) *netv1.Ingress {
+	updatedIng := updateIngress(ingressCreate.ObjectMeta.Name,
+		ingressCreate.ObjectMeta.Namespace,
+		hostName,
+		serviceName,
+		path,
+		netv1.PathTypeImplementationSpecific,
+		ingressCreate.ObjectMeta.Labels,
+		map[string]string{
+			"zalando.org/skipper-routes": ingressRoute,
+		},
+		port,
+	)
+	ingressUpdate, err := cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{})
+	framework.ExpectNoError(err)
+	time.Sleep(2 * time.Minute) // wait for routing change propagation
+
+	return ingressUpdate
+}

test/e2e/apply/secret.yaml

@@ -19,3 +19,7 @@ data:
   ETCD_SCALYR_KEY: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwETRzvm1hGplyUn23FEXUVtAAAAnjCBmwYJKoZIhvcNAQcGoIGNMIGKAgEAMIGEBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOfPJJJy60sDkZEIHgIBEIBXiANNciEqpcuZ3hFPCt6NkFtk0WBTSasDQHHbyuR8O+n5iM9k8/nUTLUrFlhba8blArq/ALE8vuKNdlS17q6PxGlvwJFFXQn/McohMpdyfnfQYKW8MPCu"
   OKTA_AUTH_ISSUER_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGmCMhSN2Er1sw2ofYnI44EAAAApDCBoQYJKoZIhvcNAQcGoIGTMIGQAgEAMIGKBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDO2IC+r/zcUzXoQEHAIBEIBdrFchwu9i7LpMbyDbslu/lBxvfyh+nCGK33jtcxT3RdxuTXWuSJhkX+gU4cgFXAI5LLnXh4M20jHUEEPU78MJWR47HLTPGPJcKQj5fOpPqpD3duuKIrZDRm5ba6AN"
   SESSION_MANAGER_DESTINATION_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwF7fOZ9i6BDvWdNEddR7LZOAAAArjCBqwYJKoZIhvcNAQcGoIGdMIGaAgEAMIGUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBJwU/Zns+mzOBgczQIBEIBn/86xpnVO2Apr5nG3waPEAGCFYDWdOXcaS7pFKdNIhpXaADtODQtEd874HcE0W2I3bjKr3d3ghJFdN8r0BZiSmTbgc0fn+5ZiBTyGBfzWP4BCzxjRMvURl/7MX8ygwL78hpSxyRypAQ=="
+  STYRA_TOKEN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGba1bnMYs8qNGHMd7a66H8AAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDE/go6xBMbsPDNgTsQIBEIB7E+VUre5YvyOJ8Apbi1SXMqNsr1rMp38is8iM5nP3LXHknTKIBZMgTeRilreIGQ25+UjWjBrTRUWmmD5jq6oN/d8t+0AEcLOpGZUplZSV+za31/3pAOPCFSqhUgzlUiI6LWW1kNKtmWwNh9aDEu4geiOAJSxJTAot6H77"
+  SKIPPER_OPA_BUCKET_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGZdCVDLsCdProfzvZU7UAwAAAAlzCBlAYJKoZIhvcNAQcGoIGGMIGDAgEAMH4GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMehOf7Uu444SWS6kbAgEQgFFPMaa0flwHLpxrkYjJMK4jXc0q4kX+KGrB5GFjKuUgOUPmQ+ME/aQduxwl2+xUilrKP50/NLXgMNHjeeHuZfoyiSgpGFBM4z8L0N6ggf2uE5U="
+  SKIPPER_OPA_OBSERVABILITY_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwHl773AuNEvIpzaM6ycpDNSAAAAqzCBqAYJKoZIhvcNAQcGoIGaMIGXAgEAMIGRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGld6jpQ38gOzVcn0gIBEIBkTHbv3adeEfRntVTUQyyQkIhUnc0QXKtmtJEdvBoRzWiJIBKQUQuM1VBV0re3HkO8HSY59nkwyHEncBMkHJoI9rC2LJuWU20oCjPw9lbweih+6Sxo+nqkDrQd+mHp+uA9Om3KqA=="
+  SKIPPER_OPA_BUNDLES_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwFnhaIRP4+3Y69xp1ycTI7qAAAAsTCBrgYJKoZIhvcNAQcGoIGgMIGdAgEAMIGXBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDF9gAl70l2g2kwfnJgIBEIBqP/DgIhIu5x5XNR1Ubqinz6r4ttQoHty8nXd6mxie2r6NxHskNOqkiSactUKhNIhboNlNsO4p4rKEkhglTeFZlEQvgEYNioWPw39xqICnUDPVr+Kp0Yrs/bzPLPV9wOlB917UiT7WJNybPg=="

test/e2e/cluster_config.sh

@@ -47,6 +47,11 @@ clusters:
     karpenter_pools_enabled: "true"
     okta_auth_client_id: "kubernetes.cluster.teapot-e2e"
     teapot_admission_controller_validate_pod_images_soft_fail_namespaces: "^kube-system$"
+    skipper_open_policy_agent_enabled: "${SKIPPER_OPA_ENABLED}"
+    skipper_open_policy_agent_styra_token: "${STYRA_TOKEN}"
+    skipper_open_policy_agent_bucket_arn: "${SKIPPER_OPA_BUCKET_ARN}"
+    skipper_open_policy_agent_observability_url: "${SKIPPER_OPA_OBSERVABILITY_URL}"
+    skipper_open_policy_agent_bundles_url: "${SKIPPER_OPA_BUNDLES_URL}"
   criticality_level: 1
   environment: e2e
   id: ${CLUSTER_ID}

test/e2e/run_e2e.sh

@@ -183,7 +183,7 @@ if [ "$e2e" = true ]; then
 
     mkdir -p junit_reports
     ginkgo -procs=25 -flake-attempts=2 \
-        -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \
+        -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\]|\[Opa\])" \
         -skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \
         "e2e.test" -- \
         -delete-namespace-on-failure=false \