diff --git a/cluster/manifests/02-kube-aws-iam-controller/deployment.yaml b/cluster/manifests/02-kube-aws-iam-controller/deployment.yaml index 5815ab3521..0de5c110c7 100644 --- a/cluster/manifests/02-kube-aws-iam-controller/deployment.yaml +++ b/cluster/manifests/02-kube-aws-iam-controller/deployment.yaml @@ -27,7 +27,7 @@ spec: hostNetwork: true containers: - name: kube-aws-iam-controller - image: container-registry.zalando.net/teapot/kube-aws-iam-controller:v0.3.0-49-g8369d21 + image: container-registry.zalando.net/teapot/kube-aws-iam-controller:v0.3.0-52-g5502e4d env: - name: AWS_DEFAULT_REGION value: "{{.Cluster.Region}}" diff --git a/cluster/manifests/03-ebs-csi/controller.yaml b/cluster/manifests/03-ebs-csi/controller.yaml index f1121e819e..fc8e30ed47 100644 --- a/cluster/manifests/03-ebs-csi/controller.yaml +++ b/cluster/manifests/03-ebs-csi/controller.yaml @@ -82,7 +82,7 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - name: csi-provisioner - image: container-registry.zalando.net/teapot/external-provisioner:v5.1.0-eks-1-31-10-master-25 + image: container-registry.zalando.net/teapot/external-provisioner:v5.1.0-eks-1-31-10-master-26 args: - --csi-address=$(ADDRESS) - --v=2 @@ -107,7 +107,7 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - name: csi-attacher - image: container-registry.zalando.net/teapot/external-attacher:v4.7.0-eks-1-31-10-master-25 + image: container-registry.zalando.net/teapot/external-attacher:v4.7.0-eks-1-31-10-master-26 args: - --csi-address=$(ADDRESS) - --v=2 @@ -129,7 +129,7 @@ spec: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - name: csi-resizer - image: container-registry.zalando.net/teapot/external-resizer:v1.12.0-eks-1-31-10-master-25 + image: container-registry.zalando.net/teapot/external-resizer:v1.12.0-eks-1-31-10-master-26 args: - --csi-address=$(ADDRESS) - --v=2 diff --git a/cluster/manifests/03-ebs-csi/node.yaml b/cluster/manifests/03-ebs-csi/node.yaml index 343201f7be..f275c656a4 100644 --- a/cluster/manifests/03-ebs-csi/node.yaml +++ b/cluster/manifests/03-ebs-csi/node.yaml @@ -34,7 +34,7 @@ spec: runAsUser: 0 containers: - name: ebs-plugin - image: container-registry.zalando.net/teapot/aws-ebs-csi-driver:v1.38.1-master-24 + image: container-registry.zalando.net/teapot/aws-ebs-csi-driver:v1.38.1-master-26 args: - node - --endpoint=$(CSI_ENDPOINT) @@ -77,7 +77,7 @@ spec: privileged: true readOnlyRootFilesystem: true - name: node-driver-registrar - image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-25 + image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-26 args: - --csi-address=$(ADDRESS) - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) diff --git a/cluster/manifests/kube-metrics-adapter/01-rbac.yaml b/cluster/manifests/kube-metrics-adapter/01-rbac.yaml index 39877bcf59..65a41981b8 100644 --- a/cluster/manifests/kube-metrics-adapter/01-rbac.yaml +++ b/cluster/manifests/kube-metrics-adapter/01-rbac.yaml @@ -169,3 +169,36 @@ subjects: - kind: ServiceAccount name: custom-metrics-apiserver namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kube-metrics-adapter-scaling-schedule-scaler + labels: + application: kubernetes + component: kube-metrics-adapter +rules: +- apiGroups: + - "apps" + resources: + - deployments/scale + - statefulsets/scale + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kube-metrics-adapter-scaling-schedule-scaler + labels: + application: kubernetes + component: kube-metrics-adapter +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-metrics-adapter-scaling-schedule-scaler +subjects: +- kind: ServiceAccount + name: custom-metrics-apiserver + namespace: kube-system diff --git a/cluster/manifests/kube-metrics-adapter/deployment.yaml b/cluster/manifests/kube-metrics-adapter/deployment.yaml index 79ea859ea2..e7fdeb15c5 100644 --- a/cluster/manifests/kube-metrics-adapter/deployment.yaml +++ b/cluster/manifests/kube-metrics-adapter/deployment.yaml @@ -27,7 +27,7 @@ spec: serviceAccountName: custom-metrics-apiserver containers: - name: kube-metrics-adapter - image: container-registry.zalando.net/teapot/kube-metrics-adapter:kube-metrics-adapter-0.2.3-46-ge580c5b + image: container-registry.zalando.net/teapot/kube-metrics-adapter:kube-metrics-adapter-0.2.3-49-gd416441 env: - name: AWS_REGION value: {{ .Cluster.Region }} diff --git a/cluster/manifests/s3-csi-driver/daemonset.yaml b/cluster/manifests/s3-csi-driver/daemonset.yaml index 4a5c21d20b..0fb3b0097f 100644 --- a/cluster/manifests/s3-csi-driver/daemonset.yaml +++ b/cluster/manifests/s3-csi-driver/daemonset.yaml @@ -95,7 +95,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-24 + image: container-registry.zalando.net/teapot/node-driver-registrar:v2.12.0-eks-1-31-10-master-26 imagePullPolicy: IfNotPresent name: node-driver-registrar resources: diff --git a/cluster/manifests/skipper/rbac.yaml b/cluster/manifests/skipper/01-rbac.yaml similarity index 100% rename from cluster/manifests/skipper/rbac.yaml rename to cluster/manifests/skipper/01-rbac.yaml diff --git a/delivery.yaml b/delivery.yaml index 91b8e61804..7158aa91b7 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -80,6 +80,8 @@ pipeline: value: "teapot-kubernetes-e2e-results" - name: ETCD_ENDPOINTS value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479" + - name: SKIPPER_OPA_ENABLED + value: "true" - name: HOSTED_ZONE valueFrom: configMapKeyRef: @@ -155,6 +157,26 @@ pipeline: secretKeyRef: name: kubernetes-e2e-config-secret key: "OKTA_AUTH_ISSUER_URL" + - name: STYRA_TOKEN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "STYRA_TOKEN" + - name: SKIPPER_OPA_BUCKET_ARN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_BUCKET_ARN" + - name: SKIPPER_OPA_OBSERVABILITY_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_OBSERVABILITY_URL" + - name: SKIPPER_OPA_BUNDLES_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_BUNDLES_URL" - name: CLUSTER_ADMIN_TOKEN valueFrom: secretKeyRef: diff --git a/test/e2e/aaas.go b/test/e2e/aaas.go new file mode 100644 index 0000000000..27995a7370 --- /dev/null +++ b/test/e2e/aaas.go @@ -0,0 +1,153 @@ +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + netv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/framework/ingress" + admissionapi "k8s.io/pod-security-admission/api" + "net/http" + "time" +) + +var _ = Describe("Ingress tests for OPA filters", func() { + f := framework.NewDefaultFramework("skipper-ingress-with-opa") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline + + var ( + cs kubernetes.Interface + jig *ingress.TestJig + ingressCreate *netv1.Ingress + ns string + hostName string + port int + serviceName = "aaas-infrastructure-tests" + opaPolicyName = "aaas-infrastructure-tests" + ) + + BeforeEach(func() { + jig = ingress.NewIngressTestJig(f.ClientSet) + cs = f.ClientSet + ns = f.Namespace.Name + hostName = fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), E2EHostedZone()) + port = 8080 + + ingressCreate = createIngressWithInfo(serviceName, hostName, ns, port, cs, jig) + }) + + It("opaAuthorizeRequest filter should pass through authorized requests [Ingress] [Opa]", func() { + // Authorization done with the rule + // https://github.bus.zalan.do/corporate-iam/aaas-infrastructure-tests-policies/blob/main/bundle/policy/ingress/rules.rego + + authorizationEnforcedPath := "/auth" + path := "/" + ingressRoute := fmt.Sprintf(`authorize: PathRegexp("%s") -> modPath("%s", "%s") -> opaAuthorizeRequest("%s") -> inlineContent("Got response") -> ;`, authorizationEnforcedPath, authorizationEnforcedPath, path, opaPolicyName) + ingressUpdate := updateIngressAndWait(serviceName, hostName, path, ingressRoute, port, ingressCreate, cs) + + By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + rt, quit := createHTTPRoundTripper() + defer func() { + quit <- struct{}{} + }() + + url := "https://" + hostName + authorizationEnforcedPath + req, err := http.NewRequest("GET", url, nil) + resp, err := getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) + + By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + req.Header.Set("Authorization", "Basic valid_token") // Authorized request + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + s, err := getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(Equal("Got response")) + }) + + It("opaServeResponse filter should return body [Ingress] [Opa]", func() { + // Authorization done with the rule + // https://github.bus.zalan.do/corporate-iam/aaas-infrastructure-tests-policies/blob/main/bundle/policy/ingress/permissions.rego + + serveResponsePath := "/permissions" + expectedPermissionsContent := "\"permissions\":{\"permission1\":{},\"permission2\":{}}" + path := serveResponsePath + ingressRoute := fmt.Sprintf(`authorize: PathRegexp("%s") -> opaServeResponse("%s") -> ;`, serveResponsePath, opaPolicyName) + ingressUpdate := updateIngressAndWait(serviceName, hostName, path, ingressRoute, port, ingressCreate, cs) + + By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 403 with opaServeResponse %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + rt, quit := createHTTPRoundTripper() + defer func() { + quit <- struct{}{} + }() + + url := "https://" + hostName + serveResponsePath + req, err := http.NewRequest("GET", url, nil) + resp, err := getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) + + By(fmt.Sprintf("Calling ingress %s/%s we wait to get a 200 with opaServeResponse %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + req.Header.Set("Authorization", "Basic permissions_token") // Authorized request + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + s, err := getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(ContainSubstring(expectedPermissionsContent)) + }) + +}) + +func createIngressWithInfo(serviceName, hostName, ns string, port int, cs kubernetes.Interface, jig *ingress.TestJig) *netv1.Ingress { + labels := map[string]string{"app": serviceName} + waitTime := 10 * time.Minute + + // Create initial ingress + ing := createIngress(serviceName, hostName, ns, "/", netv1.PathTypeImplementationSpecific, labels, nil, port) + var err error + ingressCreate, err := cs.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + // Wait for ingress address + addr, err := jig.WaitForIngressAddress(context.TODO(), cs, ns, ingressCreate.Name, waitTime) + framework.ExpectNoError(err) + + // Ensure ingress exists + _, err = cs.NetworkingV1().Ingresses(ns).Get(context.TODO(), ing.Name, metav1.GetOptions{ResourceVersion: "0"}) + framework.ExpectNoError(err) + + By("Waiting for LB to create endpoint and skipper route") + err = waitForResponse(addr, "https", waitTime, isNotFound, true) + framework.ExpectNoError(err) + + return ingressCreate +} + +func updateIngressAndWait(serviceName, hostName, path, ingressRoute string, port int, ingressCreate *netv1.Ingress, cs kubernetes.Interface) *netv1.Ingress { + updatedIng := updateIngress(ingressCreate.ObjectMeta.Name, + ingressCreate.ObjectMeta.Namespace, + hostName, + serviceName, + path, + netv1.PathTypeImplementationSpecific, + ingressCreate.ObjectMeta.Labels, + map[string]string{ + "zalando.org/skipper-routes": ingressRoute, + }, + port, + ) + ingressUpdate, err := cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + time.Sleep(2 * time.Minute) // wait for routing change propagation + + return ingressUpdate +} diff --git a/test/e2e/apply/secret.yaml b/test/e2e/apply/secret.yaml index aeda315268..8c03081bad 100644 --- a/test/e2e/apply/secret.yaml +++ b/test/e2e/apply/secret.yaml @@ -19,3 +19,7 @@ data: ETCD_SCALYR_KEY: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwETRzvm1hGplyUn23FEXUVtAAAAnjCBmwYJKoZIhvcNAQcGoIGNMIGKAgEAMIGEBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOfPJJJy60sDkZEIHgIBEIBXiANNciEqpcuZ3hFPCt6NkFtk0WBTSasDQHHbyuR8O+n5iM9k8/nUTLUrFlhba8blArq/ALE8vuKNdlS17q6PxGlvwJFFXQn/McohMpdyfnfQYKW8MPCu" OKTA_AUTH_ISSUER_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGmCMhSN2Er1sw2ofYnI44EAAAApDCBoQYJKoZIhvcNAQcGoIGTMIGQAgEAMIGKBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDO2IC+r/zcUzXoQEHAIBEIBdrFchwu9i7LpMbyDbslu/lBxvfyh+nCGK33jtcxT3RdxuTXWuSJhkX+gU4cgFXAI5LLnXh4M20jHUEEPU78MJWR47HLTPGPJcKQj5fOpPqpD3duuKIrZDRm5ba6AN" SESSION_MANAGER_DESTINATION_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwF7fOZ9i6BDvWdNEddR7LZOAAAArjCBqwYJKoZIhvcNAQcGoIGdMIGaAgEAMIGUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBJwU/Zns+mzOBgczQIBEIBn/86xpnVO2Apr5nG3waPEAGCFYDWdOXcaS7pFKdNIhpXaADtODQtEd874HcE0W2I3bjKr3d3ghJFdN8r0BZiSmTbgc0fn+5ZiBTyGBfzWP4BCzxjRMvURl/7MX8ygwL78hpSxyRypAQ==" + STYRA_TOKEN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGba1bnMYs8qNGHMd7a66H8AAAAwjCBvwYJKoZIhvcNAQcGoIGxMIGuAgEAMIGoBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDE/go6xBMbsPDNgTsQIBEIB7E+VUre5YvyOJ8Apbi1SXMqNsr1rMp38is8iM5nP3LXHknTKIBZMgTeRilreIGQ25+UjWjBrTRUWmmD5jq6oN/d8t+0AEcLOpGZUplZSV+za31/3pAOPCFSqhUgzlUiI6LWW1kNKtmWwNh9aDEu4geiOAJSxJTAot6H77" + SKIPPER_OPA_BUCKET_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGZdCVDLsCdProfzvZU7UAwAAAAlzCBlAYJKoZIhvcNAQcGoIGGMIGDAgEAMH4GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMehOf7Uu444SWS6kbAgEQgFFPMaa0flwHLpxrkYjJMK4jXc0q4kX+KGrB5GFjKuUgOUPmQ+ME/aQduxwl2+xUilrKP50/NLXgMNHjeeHuZfoyiSgpGFBM4z8L0N6ggf2uE5U=" + SKIPPER_OPA_OBSERVABILITY_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwHl773AuNEvIpzaM6ycpDNSAAAAqzCBqAYJKoZIhvcNAQcGoIGaMIGXAgEAMIGRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGld6jpQ38gOzVcn0gIBEIBkTHbv3adeEfRntVTUQyyQkIhUnc0QXKtmtJEdvBoRzWiJIBKQUQuM1VBV0re3HkO8HSY59nkwyHEncBMkHJoI9rC2LJuWU20oCjPw9lbweih+6Sxo+nqkDrQd+mHp+uA9Om3KqA==" + SKIPPER_OPA_BUNDLES_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwFnhaIRP4+3Y69xp1ycTI7qAAAAsTCBrgYJKoZIhvcNAQcGoIGgMIGdAgEAMIGXBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDF9gAl70l2g2kwfnJgIBEIBqP/DgIhIu5x5XNR1Ubqinz6r4ttQoHty8nXd6mxie2r6NxHskNOqkiSactUKhNIhboNlNsO4p4rKEkhglTeFZlEQvgEYNioWPw39xqICnUDPVr+Kp0Yrs/bzPLPV9wOlB917UiT7WJNybPg==" diff --git a/test/e2e/cluster_config.sh b/test/e2e/cluster_config.sh index 36869500cb..8b61d49646 100755 --- a/test/e2e/cluster_config.sh +++ b/test/e2e/cluster_config.sh @@ -47,6 +47,11 @@ clusters: karpenter_pools_enabled: "true" okta_auth_client_id: "kubernetes.cluster.teapot-e2e" teapot_admission_controller_validate_pod_images_soft_fail_namespaces: "^kube-system$" + skipper_open_policy_agent_enabled: "${SKIPPER_OPA_ENABLED}" + skipper_open_policy_agent_styra_token: "${STYRA_TOKEN}" + skipper_open_policy_agent_bucket_arn: "${SKIPPER_OPA_BUCKET_ARN}" + skipper_open_policy_agent_observability_url: "${SKIPPER_OPA_OBSERVABILITY_URL}" + skipper_open_policy_agent_bundles_url: "${SKIPPER_OPA_BUNDLES_URL}" criticality_level: 1 environment: e2e id: ${CLUSTER_ID} diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index eb8b2f5538..5a520b1545 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -183,7 +183,7 @@ if [ "$e2e" = true ]; then mkdir -p junit_reports ginkgo -procs=25 -flake-attempts=2 \ - -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \ + -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\]|\[Opa\])" \ -skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \ "e2e.test" -- \ -delete-namespace-on-failure=false \