test(e2e): add Cilium-CNI e2e suite #10
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| #!/usr/bin/env bash | ||
| # shellcheck disable=SC1091 | ||
| . lib.sh | ||
|
|
||
| # Cilium-CNI counterpart of e2e/cross-mesh.sh. The Kilo DaemonSet is the | ||
| # one applied by create_cilium_cluster (kilo-kind-cilium.yaml), which | ||
| # already runs Kilo with --cni=false --compatibility=cilium. This suite | ||
| # only annotates locations and switches granularity to "cross". | ||
| setup_suite() { | ||
| _kubectl annotate node "$KIND_CLUSTER-control-plane" kilo.squat.ai/location=loc-a --overwrite | ||
| _kubectl annotate node "$KIND_CLUSTER-worker" kilo.squat.ai/location=loc-a --overwrite | ||
| _kubectl annotate node "$KIND_CLUSTER-worker2" kilo.squat.ai/location=loc-b --overwrite | ||
| # shellcheck disable=SC2016 | ||
| _kubectl patch ds -n kube-system kilo -p '{"spec":{"template":{"spec":{"containers":[{"name":"kilo","args":["--hostname=$(NODE_NAME)","--create-interface=false","--cni=false","--compatibility=cilium","--mesh-granularity=cross","--kubeconfig=/etc/kubernetes/kubeconfig","--internal-cidr=$(NODE_IP)/32"]}]}}}}' | ||
| block_until_ready_by_name kube-system kilo-userspace | ||
| } | ||
|
|
||
| test_cilium_cross_mesh_connectivity() { | ||
| assert "retry 30 5 '' check_ping" "should be able to ping all Pods over Cilium VXLAN + Kilo cross mesh" | ||
| assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings" | ||
| echo "sleep for 30s (one reconciliation period) and try again..." | ||
| sleep 30 | ||
| assert "retry 10 5 'the adjacency matrix is not complete yet' check_adjacent 3" "adjacency should return the right number of successful pings after reconciling" | ||
| } | ||
|
|
||
| test_cilium_cross_peer_topology() { | ||
| local CP_PEERS WORKER_PEERS WORKER2_PEERS | ||
| CP_PEERS=$(_kgctl showconf node "$KIND_CLUSTER-control-plane" | grep -c '^\[Peer\]') | ||
| WORKER_PEERS=$(_kgctl showconf node "$KIND_CLUSTER-worker" | grep -c '^\[Peer\]') | ||
| WORKER2_PEERS=$(_kgctl showconf node "$KIND_CLUSTER-worker2" | grep -c '^\[Peer\]') | ||
| assert_equals "1" "$CP_PEERS" "control-plane (loc-a) should have 1 peer (the loc-b node)" | ||
| assert_equals "1" "$WORKER_PEERS" "worker (loc-a) should have 1 peer (the loc-b node)" | ||
| assert_equals "2" "$WORKER2_PEERS" "worker2 (loc-b) should have 2 peers (both loc-a nodes)" | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| #!/usr/bin/env bash | ||
| # shellcheck disable=SC1091 | ||
| . lib.sh | ||
|
|
||
| # Bring up a kind cluster with Cilium as the CNI for the Cilium-mode e2e | ||
| # suite. Counterpart of e2e/setup.sh, which provisions a cluster that | ||
| # uses the Kilo bridge CNI. | ||
| setup_suite() { | ||
| create_cilium_cluster "$(build_kind_config 2)" | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| #!/usr/bin/env bash | ||
| # shellcheck disable=SC1091 | ||
| . lib.sh | ||
|
|
||
| teardown_suite () { | ||
| if [ -n "$E2E_SKIP_TEARDOWN_ON_FAILURE" ]; then | ||
| return | ||
| fi | ||
| delete_cluster | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,146 @@ | ||
| apiVersion: v1 | ||
| kind: ServiceAccount | ||
| metadata: | ||
| name: kilo | ||
| namespace: kube-system | ||
| --- | ||
| apiVersion: rbac.authorization.k8s.io/v1 | ||
| kind: ClusterRole | ||
| metadata: | ||
| name: kilo | ||
| rules: | ||
| - apiGroups: | ||
| - "" | ||
| resources: | ||
| - nodes | ||
| verbs: | ||
| - list | ||
| - patch | ||
| - watch | ||
| - apiGroups: | ||
| - kilo.squat.ai | ||
| resources: | ||
| - peers | ||
| verbs: | ||
| - list | ||
| - watch | ||
| - apiGroups: | ||
| - apiextensions.k8s.io | ||
| resources: | ||
| - customresourcedefinitions | ||
| verbs: | ||
| - get | ||
| --- | ||
| apiVersion: rbac.authorization.k8s.io/v1 | ||
| kind: ClusterRoleBinding | ||
| metadata: | ||
| name: kilo | ||
| roleRef: | ||
| apiGroup: rbac.authorization.k8s.io | ||
| kind: ClusterRole | ||
| name: kilo | ||
| subjects: | ||
| - kind: ServiceAccount | ||
| name: kilo | ||
| namespace: kube-system | ||
| --- | ||
| # Kilo DaemonSet for the Cilium e2e suite. The CNI is provided by Cilium | ||
| # (no kilo CNI ConfigMap and no install-cni init container), so Kilo runs | ||
| # in --cni=false / --compatibility=cilium mode and only manages the WG mesh | ||
| # on top of Cilium's overlay. | ||
| apiVersion: apps/v1 | ||
| kind: DaemonSet | ||
| metadata: | ||
| name: kilo | ||
| namespace: kube-system | ||
| labels: | ||
| app.kubernetes.io/name: kilo-userspace | ||
| app.kubernetes.io/part-of: kilo | ||
| spec: | ||
| selector: | ||
| matchLabels: | ||
| app.kubernetes.io/name: kilo-userspace | ||
| app.kubernetes.io/part-of: kilo | ||
| template: | ||
| metadata: | ||
| labels: | ||
| app.kubernetes.io/name: kilo-userspace | ||
| app.kubernetes.io/part-of: kilo | ||
| spec: | ||
| serviceAccountName: kilo | ||
| hostNetwork: true | ||
| containers: | ||
| - name: kilo | ||
| image: squat/kilo:test | ||
| imagePullPolicy: Never | ||
| args: | ||
| - --hostname=$(NODE_NAME) | ||
| - --create-interface=false | ||
| - --cni=false | ||
| - --compatibility=cilium | ||
| - --mesh-granularity=full | ||
| - --kubeconfig=/etc/kubernetes/kubeconfig | ||
| - --internal-cidr=$(NODE_IP)/32 | ||
| env: | ||
| - name: NODE_NAME | ||
| valueFrom: | ||
| fieldRef: | ||
| fieldPath: spec.nodeName | ||
| - name: NODE_IP | ||
| valueFrom: | ||
| fieldRef: | ||
| fieldPath: status.hostIP | ||
| ports: | ||
| - containerPort: 1107 | ||
| name: metrics | ||
| securityContext: | ||
| privileged: true | ||
| volumeMounts: | ||
| - name: kilo-dir | ||
| mountPath: /var/lib/kilo | ||
| - name: lib-modules | ||
| mountPath: /lib/modules | ||
| readOnly: true | ||
| - name: xtables-lock | ||
| mountPath: /run/xtables.lock | ||
| readOnly: false | ||
| - name: wireguard | ||
| mountPath: /var/run/wireguard | ||
| readOnly: false | ||
| - name: kubeconfig | ||
| mountPath: /etc/kubernetes | ||
| readOnly: true | ||
| - name: wireguard | ||
| image: ghcr.io/masipcat/wireguard-go-docker:0.0.20230223 | ||
| args: | ||
| - wireguard-go | ||
| - --foreground | ||
| - kilo0 | ||
| securityContext: | ||
| privileged: true | ||
| volumeMounts: | ||
| - name: wireguard | ||
| mountPath: /var/run/wireguard | ||
| readOnly: false | ||
| tolerations: | ||
| - effect: NoSchedule | ||
| operator: Exists | ||
| - effect: NoExecute | ||
| operator: Exists | ||
| volumes: | ||
| - name: kilo-dir | ||
| hostPath: | ||
| path: /var/lib/kilo | ||
| - name: lib-modules | ||
| hostPath: | ||
| path: /lib/modules | ||
| - name: xtables-lock | ||
| hostPath: | ||
| path: /run/xtables.lock | ||
| type: FileOrCreate | ||
| - name: wireguard | ||
| hostPath: | ||
| path: /var/run/wireguard | ||
| - name: kubeconfig | ||
| secret: | ||
| secretName: kubeconfig |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Using a hardcoded
sleepin end-to-end tests is generally discouraged as it can lead to flaky tests (if the reconciliation takes longer than 30s) or unnecessarily slow tests (if it takes much less). It is better to poll for a specific state or condition that indicates the reconciliation has completed, such as checking for a specific log message or a change in the node/peer status.