Test serverless with strict central network policies (#1533)

Co-authored-by: Filip Strozik <[email protected]>

Author: kwiatekus

.github/actions/with-fixture/action.yaml

@@ -0,0 +1,16 @@
+name: 'Apply extra test fixture'
+description: 'Apply extra test fixture'
+
+inputs:
+  fixture_path:
+    description: 'Path pointing to a folder containing manifests to apply (relative to project root)'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Apply fixture
+      run: |
+        kubectl apply -f ${{ inputs.fixture_path }}
+      shell: bash
+

.github/workflows/_integration-tests.yaml

@@ -24,6 +24,9 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
       - uses: ./.github/actions/setup-go
       - uses: ./.github/actions/create-k3d-cluster
+      - uses: actions/with-fixture
+        with:
+          fixture_path: tests/with-fixture/strict-network-policies
       - name: run test
         run: |
           make -C components/operator deploy
@@ -43,6 +46,9 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
       - uses: ./.github/actions/setup-go
       - uses: ./.github/actions/create-k3d-cluster
+      - uses: actions/with-fixture
+        with:
+          fixture_path: tests/with-fixture/strict-network-policies
       - name: run test
         run: |
           make install-buildless-serverless-custom-operator
@@ -61,6 +67,9 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
       - uses: ./.github/actions/setup-go
       - uses: ./.github/actions/create-k3d-cluster
+      - uses: actions/with-fixture
+        with:
+          fixture_path: tests/with-fixture/strict-network-policies
       - name: run test
         run: |
           make install-serverless-custom-operator
@@ -79,6 +88,9 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
       - uses: ./.github/actions/setup-go
       - uses: ./.github/actions/create-k3d-cluster
+      - uses: actions/with-fixture
+        with:
+          fixture_path: tests/with-fixture/strict-network-policies
       - name: run tests
         run: |
           make install-serverless-custom-operator
@@ -116,6 +128,9 @@ jobs:
           btp_kyma_plan: '${{ secrets.BTP_KYMA_PLAN }}'
           btp_kyma_modules: "[]"
           btp_kyma_autoscaler_min: 4
+      - uses: actions/with-fixture
+        with:
+          fixture_path: tests/with-fixture/strict-network-policies
       - name: run tests
         run: |
           make install-serverless-custom-operator

components/operator/controllers/serverless_controller_rbac.go

@@ -36,3 +36,5 @@ package controllers
 
 //+kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete;deletecollection
 //+kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=get;list;watch;create;update;patch;delete;deletecollection
+
+//+kubebuilder:rbac:groups=networking.k8s.io,resources=networkpolicies,verbs=get;list;watch;create;update;patch;delete;deletecollection

config/buildless-serverless/templates/deployment.yaml

@@ -20,6 +20,9 @@ spec:
         sidecar.istio.io/inject: "false"
       labels:
         control-plane: controller-manager
+        networking.kyma-project.io/to-apiserver: allowed
+        networking.kyma-project.io/from-serverless: allowed
+        kyma-project.io/module: serverless
     spec:
       containers:
         - args:

config/buildless-serverless/templates/egress-network-policy.yaml

@@ -0,0 +1,14 @@
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: serverless.kyma-project.io--allow-all-egress-from-controllers
+spec:
+  podSelector:
+    matchLabels:
+      kyma-project.io/module: serverless
+      networking.kyma-project.io/from-serverless: allowed
+  policyTypes:
+  - Egress
+  egress:
+    - {}

config/operator/base/deployment/deployment.yaml

@@ -21,6 +21,7 @@ spec:
         control-plane: operator
         sidecar.istio.io/inject: "false"
         app.kubernetes.io/component: serverless-operator.kyma-project.io
+        networking.kyma-project.io/to-apiserver: allowed
     spec:
       priorityClassName: "operator-priority"
       securityContext:

config/operator/base/rbac/role.yaml

@@ -138,6 +138,19 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - networkpolicies
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - operator.kyma-project.io
   resources:

config/serverless/charts/webhook/templates/deployment.yaml

@@ -31,6 +31,7 @@ spec:
       {{- end }}
       labels:
         {{- include "tplValue" ( dict "value" .Values.commonLabels "context" . ) | nindent 8 }}
+        networking.kyma-project.io/from-seed: allowed
     spec:
       serviceAccountName: {{ template "webhook.fullname" . }}
       volumes:

config/serverless/templates/egress-network-policy.yaml

@@ -0,0 +1,14 @@
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ template "fullname" . }}.kyma-project.io--allow-all-egress-from-controllers
+spec:
+  podSelector:
+    matchLabels:
+      kyma-project.io/module: serverless
+      networking.kyma-project.io/from-serverless: allowed
+  policyTypes:
+  - Egress
+  egress:
+    - {}

config/serverless/templates/ingress-network-policy.yaml

@@ -0,0 +1,17 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ template "fullname" . }}.kyma-project.io--allow-to-registry-port
+spec:
+  podSelector:
+    matchLabels:
+      kyma-project.io/module: {{ template "fullname" . }}
+      app.kubernetes.io/name: docker-registry
+  policyTypes:
+  - Ingress
+  ingress:
+  - from:
+    ports:
+    - protocol: TCP
+      port: {{ .Values.global.registryServicePort }}

config/serverless/values.yaml

@@ -129,7 +129,9 @@ deployment:
     timeoutSeconds: 10
     periodSeconds: 30
 pod:
-  labels: {}
+  labels: 
+    networking.kyma-project.io/to-apiserver: allowed
+    networking.kyma-project.io/from-serverless: allowed
   annotations:
     sidecar.istio.io/inject: "false"
     prometheus.io/scrape: "false"

tests/serverless/internal/resources/git/gitserver.go

@@ -3,6 +3,7 @@ package git
 import (
 	"context"
 	"fmt"
+
 	"github.com/kyma-project/serverless/tests/serverless/internal/resources"
 	"github.com/kyma-project/serverless/tests/serverless/internal/resources/app"
 	"github.com/kyma-project/serverless/tests/serverless/internal/utils"

tests/serverless/internal/resources/networkpolicy/networkpolicy.go

@@ -0,0 +1,78 @@
+package networkpolicy
+
+import (
+	"context"
+
+	"github.com/kyma-project/serverless/tests/serverless/internal/utils"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	networkingclient "k8s.io/client-go/kubernetes/typed/networking/v1"
+)
+
+const (
+	componentLabel = "component"
+)
+
+type NetworkPolicy struct {
+	name          string
+	namespace     string
+	networkingCli networkingclient.NetworkPolicyInterface
+	log           *logrus.Entry
+	spec          networkingv1.NetworkPolicySpec
+}
+
+func NewNetworkPolicy(name, namespace string, spec networkingv1.NetworkPolicySpec, networkPolicies networkingclient.NetworkPolicyInterface, log *logrus.Entry) NetworkPolicy {
+	return NetworkPolicy{
+		name:          name,
+		namespace:     namespace,
+		networkingCli: networkPolicies,
+		log:           log,
+		spec:          spec,
+	}
+}
+
+func (n NetworkPolicy) Create(spec networkingv1.NetworkPolicySpec) error {
+
+	networkPolicy := &networkingv1.NetworkPolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      n.name,
+			Namespace: n.namespace,
+			Labels: map[string]string{
+				componentLabel: n.name,
+			},
+		},
+		Spec: spec,
+	}
+	_, err := n.networkingCli.Create(context.Background(), networkPolicy, metav1.CreateOptions{})
+	return errors.Wrapf(err, "while creating NetworkPolicy %s in namespace %s", n.name, n.namespace)
+}
+
+func (n NetworkPolicy) Delete() error {
+	return n.networkingCli.Delete(context.Background(), n.name, metav1.DeleteOptions{})
+}
+
+func (n NetworkPolicy) LogResource() error {
+	policy, err := n.Get()
+	if err != nil {
+		return err
+	}
+
+	out, err := utils.PrettyMarshall(policy)
+	if err != nil {
+		return err
+	}
+
+	n.log.Infof("%s", out)
+	return nil
+}
+
+func (n NetworkPolicy) Get() (*networkingv1.NetworkPolicy, error) {
+	u, err := n.networkingCli.Get(context.Background(), n.name, metav1.GetOptions{})
+	if err != nil {
+		return nil, errors.Wrapf(err, "while getting %s", n.name)
+	}
+
+	return u, nil
+}

tests/serverless/internal/resources/networkpolicy/steps.go

@@ -0,0 +1,102 @@
+package networkpolicy
+
+import (
+	"github.com/kyma-project/serverless/tests/serverless/internal/executor"
+	"github.com/sirupsen/logrus"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	networkingclient "k8s.io/client-go/kubernetes/typed/networking/v1"
+)
+
+type newNetworkPolicyStep struct {
+	name            string
+	namespace       string
+	networkPolicies []*NetworkPolicy
+	// resCli          *resources.Resource
+	log *logrus.Entry
+}
+
+// Cleanup implements executor.Step.
+func (n newNetworkPolicyStep) Cleanup() error {
+	for _, networkPolicy := range n.networkPolicies {
+		err := networkPolicy.Delete()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Name implements executor.Step.
+func (n newNetworkPolicyStep) Name() string {
+	return n.name
+}
+
+// OnError implements executor.Step.
+func (n newNetworkPolicyStep) OnError() error {
+	for _, networkPolicy := range n.networkPolicies {
+		err := networkPolicy.LogResource()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Run implements executor.Step.
+func (n newNetworkPolicyStep) Run() error {
+	for _, networkPolicy := range n.networkPolicies {
+		err := networkPolicy.Create(networkPolicy.spec)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+var _ executor.Step = newNetworkPolicyStep{}
+
+func CreateNetworkPoliciesStep(log *logrus.Entry, name, namespace string, networkCli networkingclient.NetworkPolicyInterface) executor.Step {
+
+	allowEgressFromMockSpec := networkingv1.NetworkPolicySpec{
+		PodSelector: metav1.LabelSelector{
+			MatchLabels: map[string]string{
+				"serverless.kyma-project.io/managed-by": "function-controller",
+			},
+		},
+		PolicyTypes: []networkingv1.PolicyType{
+			networkingv1.PolicyTypeEgress,
+		},
+		Egress: []networkingv1.NetworkPolicyEgressRule{
+			networkingv1.NetworkPolicyEgressRule{},
+		},
+	}
+
+	allowIngressToMockSpec := networkingv1.NetworkPolicySpec{
+		PodSelector: metav1.LabelSelector{
+			MatchLabels: map[string]string{
+				"app.kubernetes.io/name": "eventing-publisher-proxy",
+			},
+		},
+		PolicyTypes: []networkingv1.PolicyType{
+			networkingv1.PolicyTypeIngress,
+		},
+		Ingress: []networkingv1.NetworkPolicyIngressRule{
+			networkingv1.NetworkPolicyIngressRule{},
+		},
+	}
+
+	allowEgressFromMock := NewNetworkPolicy("allow-all-egress-from-eventing-mock", "kyma-system", allowEgressFromMockSpec, networkCli, log)
+	allowIngressToMock := NewNetworkPolicy("allow-all-ingress-from-eventing-mock", "kyma-system", allowIngressToMockSpec, networkCli, log)
+
+	return newNetworkPolicyStep{
+		name:      name,
+		namespace: namespace,
+		networkPolicies: []*NetworkPolicy{
+			&allowEgressFromMock,
+			&allowIngressToMock,
+		},
+		log: log,
+	}
+
+}

tests/serverless/internal/resources/runtimes/python.go

@@ -153,6 +153,9 @@ def main(event, context):
 			},
 		},
 		Env: []v1.EnvVar{},
+		Labels: map[string]string{
+			"app.kubernetes.io/name": "eventing-publisher-proxy",
+		},
 		ResourceConfiguration: &serverlessv1alpha2.ResourceConfiguration{
 			Function: &serverlessv1alpha2.ResourceRequirements{
 				Profile: "L",