Skip to content

Commit 82a7848

Browse files
mikenairnmikenairn
committed
DNSPolicy scale test
Adds a DNSPolicy specific scale test using kube burner. The workload will create multiple instances of the dns operator in separate namespaces(kuadrant-dns-operator-x), and multiple test namespaces (scale-test-x) that the corresponding dns operator is configured to watch. The number of dns operator instances and test namespaces created is determined by the `JOB_ITERATIONS` environment variable. In each test namespace a test app and service is deployed and one or more gateways are created determined by the `NUM_GWS` environment variable. The number of listeners added to the gateway is determined by the `NUM_LISTENERS` environment variable. Each listener hostname is generated using the listener number and the `KUADRANT_ZONE_ROOT_DOMAIN` environment variable. In each test namespace a dns provider credential is created, the type created is determined by the `DNS_PROVIDER` environment variable, additional environment variables may need to be set depending on the provider type. Signed-off-by: Michael Nairn <[email protected]>
1 parent 76366ce commit 82a7848

File tree

7 files changed

+412
-3
lines changed

7 files changed

+412
-3
lines changed

Diff for: ‎Makefile

+48-3
Original file line numberDiff line numberDiff line change
@@ -102,9 +102,9 @@ reportportal: ## Upload results to reportportal. Appropriate variables for juni2
102102
reportportal: polish-junit
103103
$(RUNSCRIPT)junit2reportportal $(resultsdir)/junit-*.xml
104104

105-
# Check http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
106-
help: ## Print this help
107-
@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
105+
.PHONY: help
106+
help: ## Display this help.
107+
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
108108

109109
CR_NAMES = $\
110110
authorinos.operator.authorino.kuadrant.io,$\
@@ -138,3 +138,48 @@ clean: ## Clean all objects on cluster created by running this testsuite. Set th
138138
| xargs --no-run-if-empty -P 20 -n 1 kubectl delete --ignore-not-found -n kuadrant
139139
# this ensures dependent target is run everytime
140140
FORCE:
141+
142+
##@ Scale Testing
143+
144+
.PHONY: test-scale-dnspolicy
145+
test-scale-dnspolicy: export DNS_OPERATOR_GITHUB_ORG := kuadrant
146+
test-scale-dnspolicy: export DNS_OPERATOR_GITREF := main
147+
test-scale-dnspolicy: export JOB_ITERATIONS := 1
148+
test-scale-dnspolicy: export KUADRANT_ZONE_ROOT_DOMAIN := kuadrant.local
149+
test-scale-dnspolicy: export DNS_PROVIDER := inmemory
150+
test-scale-dnspolicy: export PROMETHEUS_URL := http://127.0.0.1:9090
151+
test-scale-dnspolicy: export PROMETHEUS_TOKEN := ""
152+
test-scale-dnspolicy: export SKIP_CLEANUP := false
153+
test-scale-dnspolicy: export NUM_GWS := 1
154+
test-scale-dnspolicy: export NUM_LISTENERS := 1
155+
test-scale-dnspolicy: KUBEBURNER_WORKLOAD := namespaced-dns-operator-deployments.yaml
156+
test-scale-dnspolicy: kube-burner ## Run DNSPolicy scale tests.
157+
@echo "test-scale-dnspolicy: KUBEBURNER_WORKLOAD=${KUBEBURNER_WORKLOAD} JOB_ITERATIONS=${JOB_ITERATIONS} KUADRANT_ZONE_ROOT_DOMAIN=${KUADRANT_ZONE_ROOT_DOMAIN} DNS_PROVIDER=${DNS_PROVIDER} PROMETHEUS_URL=${PROMETHEUS_URL} PROMETHEUS_TOKEN=${PROMETHEUS_TOKEN}"
158+
cd scale_test/dnspolicy && $(KUBE_BURNER) init -c ${KUBEBURNER_WORKLOAD} --log-level debug
159+
160+
##@ Build Dependencies
161+
162+
## Location to install dependencies to
163+
LOCALBIN ?= $(shell pwd)/bin
164+
$(LOCALBIN):
165+
mkdir -p $(LOCALBIN)
166+
167+
## Tool Binaries
168+
KUBE_BURNER ?= $(LOCALBIN)/kube-burner
169+
170+
## Tool Versions
171+
KUBE_BURNER_VERSION = v1.11.1
172+
173+
.PHONY: kube-burner
174+
kube-burner: $(KUBE_BURNER) ## Download kube-burner locally if necessary.
175+
$(KUBE_BURNER):
176+
@{ \
177+
set -e ;\
178+
mkdir -p $(dir $(KUBE_BURNER)) ;\
179+
OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \
180+
wget -O kube-burner.tar.gz https://github.com/kube-burner/kube-burner/releases/download/v1.11.1/kube-burner-V1.11.1-linux-x86_64.tar.gz ;\
181+
tar -zxvf kube-burner.tar.gz ;\
182+
mv kube-burner $(KUBE_BURNER) ;\
183+
chmod +x $(KUBE_BURNER) ;\
184+
rm -rf $${OS}-$${ARCH} kube-burner.tar.gz ;\
185+
}

Diff for: ‎scale_test/dnspolicy/README.md

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# DNSPolicy Scale Testing
2+
3+
Scale testing using [kube-burner](https://kube-burner.github.io/kube-burner/latest).
4+
5+
## Setup local environment (kind)
6+
7+
Create a kind cluster with kuadrant and prometheus/thanos installed and configured using the [kuadrant-operator](https://github.com/Kuadrant/kuadrant-operator) repo tasks and config.
8+
9+
In the kuadrant-operator directory, run the following to create a local kind cluster with kuadrant deployed:
10+
```shell
11+
make local-setup SUBNET_OFFSET=1 CIDR=26 NUM_IPS=64
12+
```
13+
Note: In order to test at scale, metallb must be configured with enough ip address to assign one to each gateway to be created.
14+
15+
## Pre-test setup
16+
17+
Scale down the default dns operator:
18+
```shell
19+
kubectl scale deployment/dns-operator-controller-manager --replicas=0 -n kuadrant-system
20+
```
21+
Note: This is required for the default kubeburner workload (namespaced-dns-operator-deployments.yaml) as it creates its own dns operator deployments.
22+
23+
Deploy the observability stack:
24+
```shell
25+
kubectl apply --server-side -k github.com/kuadrant/dns-operator/config/observability?ref=main # Run twice if it fails the first time dut o CRDs i.e. "ensure CRDs are installed first"
26+
```
27+
Note: This should be in the kuadrant-operator repo instead of the dns operator
28+
29+
Forward port for prometheus to allow kube burner to access it on the default PROMETHEUS_URL (http://127.0.0.1:9090):
30+
```shell
31+
kubectl -n monitoring port-forward service/thanos-query 9090:9090
32+
```
33+
34+
## Run test
35+
36+
The `test-scale-dnspolicy` make target can be used without input to run the default test workload with the default configuration:
37+
38+
```shell
39+
make test-scale-dnspolicy
40+
```
41+
42+
View created resources:
43+
```shell
44+
kubectl get deployments,gateways,httproutes,secrets,dnspolicy -A -l kube-burner-uuid=3e6516a7-76a7-4e9f-a995-480f3512a8bb
45+
NAMESPACE NAME READY UP-TO-DATE AVAILABLE AGE
46+
kuadrant-dns-operator-0 deployment.apps/controller-manager 1/1 1 1 14m
47+
scale-test-0 deployment.apps/gw1-i0-istio 1/1 1 1 13m
48+
scale-test-0 deployment.apps/httpbin 1/1 1 1 14m
49+
50+
NAMESPACE NAME CLASS ADDRESS PROGRAMMED AGE
51+
scale-test-0 gateway.gateway.networking.k8s.io/gw1-i0 istio 172.18.0.18 True 13m
52+
53+
NAMESPACE NAME HOSTNAMES AGE
54+
scale-test-0 httproute.gateway.networking.k8s.io/httproute-gw1-l1-i0 ["api-1.dnspolicy-scale-test-loadbalanced.kuadrant.local"] 13m
55+
56+
NAMESPACE NAME TYPE DATA AGE
57+
scale-test-0 secret/inmemory-credentials kuadrant.io/inmemory 1 14m
58+
```
59+
Note: DNSPolices are deleted as part of the run as `SKIP_CLEANUP` defaults to false
60+
61+
Alternatively it can be executed passing in values as required. Please refer to the `test-scale-dnspolicy` make target for possible variables and their default values.
62+
```shell
63+
make test-scale-dnspolicy JOB_ITERATIONS=1 NUM_GWS=1 NUM_LISTENERS=1 SKIP_CLEANUP=true DNS_PROVIDER=aws KUADRANT_ZONE_ROOT_DOMAIN=my.domain.com
64+
```
65+
66+
## Workloads
67+
68+
The following describes the behaviour and configuration of the current DNSPolicy workloads.
69+
70+
### namespaced-dns-operator-deployments
71+
72+
The workload will create multiple instances of the dns operator in separate namespaces(kuadrant-dns-operator-x), and multiple test namespaces (scale-test-x) that the corresponding dns operator is configured to watch.
73+
The number of dns operator instances and test namespaces created is determined by the `JOB_ITERATIONS` environment variable.
74+
In each test namespace a test app and service is deployed and one or more gateways are created determined by the `NUM_GWS` environment variable.
75+
The number of listeners added to the gateway is determined by the `NUM_LISTENERS` environment variable.
76+
Each listener hostname is generated using the listener number and the `KUADRANT_ZONE_ROOT_DOMAIN` environment variable.
77+
In each test namespace a dns provider credential is created, the type created is determined by the `DNS_PROVIDER` environment variable, additional environment variables may need to be set depending on the provider type.
78+
79+
#### Example Run
80+
81+
Create a shared recordset in AWS (DNS_PROVIDER=aws) for a single host with four distinct A record values, owned by four DNSRecord resources, created by four gateway/dnspolices (JOB_ITERATIONS * NUM_GWS) processed by two dns operators(JOB_ITERATIONS).
82+
83+
```shell
84+
make test-scale-dnspolicy JOB_ITERATIONS=2 NUM_GWS=2 NUM_LISTENERS=1 DNS_PROVIDER=aws KUADRANT_AWS_ACCESS_KEY_ID=<my aws access key> KUADRANT_AWS_SECRET_ACCESS_KEY=<my aws secret id>. KUADRANT_AWS_REGION='' KUADRANT_ZONE_ROOT_DOMAIN=mn.hcpapps.net SKIP_CLEANUP=true
85+
...
86+
time="2025-01-13 10:19:27" level=info msg="Finished execution with UUID: 469d4b4e-6c41-4433-8c4b-2c48ea4973bc" file="job.go:247"
87+
time="2025-01-13 10:19:27" level=info msg="👋 Exiting kube-burner 469d4b4e-6c41-4433-8c4b-2c48ea4973bc" file="kube-burner.go:85"
88+
```
89+
90+
Resource Created in cluster:
91+
92+
```shell
93+
kubectl get deployments,gateways,httproutes,secrets,dnspolicy -A -l kube-burner-uuid=469d4b4e-6c41-4433-8c4b-2c48ea4973bc
94+
NAMESPACE NAME READY UP-TO-DATE AVAILABLE AGE
95+
kuadrant-dns-operator-0 deployment.apps/controller-manager 1/1 1 1 12m
96+
kuadrant-dns-operator-1 deployment.apps/controller-manager 1/1 1 1 12m
97+
scale-test-0 deployment.apps/gw1-i0-istio 1/1 1 1 9m26s
98+
scale-test-0 deployment.apps/gw2-i0-istio 1/1 1 1 9m26s
99+
scale-test-0 deployment.apps/httpbin 1/1 1 1 11m
100+
scale-test-1 deployment.apps/gw1-i1-istio 1/1 1 1 9m26s
101+
scale-test-1 deployment.apps/gw2-i1-istio 1/1 1 1 9m25s
102+
scale-test-1 deployment.apps/httpbin 1/1 1 1 11m
103+
104+
NAMESPACE NAME CLASS ADDRESS PROGRAMMED AGE
105+
scale-test-0 gateway.gateway.networking.k8s.io/gw1-i0 istio 172.18.0.2 True 9m26s
106+
scale-test-0 gateway.gateway.networking.k8s.io/gw2-i0 istio 172.18.0.3 True 9m26s
107+
scale-test-1 gateway.gateway.networking.k8s.io/gw1-i1 istio 172.18.0.4 True 9m26s
108+
scale-test-1 gateway.gateway.networking.k8s.io/gw2-i1 istio 172.18.0.5 True 9m25s
109+
110+
NAMESPACE NAME HOSTNAMES AGE
111+
scale-test-0 httproute.gateway.networking.k8s.io/httproute-gw1-l1-i0 ["api-1.dnspolicy-scale-test-loadbalanced.mn.hcpapps.net"] 9m26s
112+
scale-test-0 httproute.gateway.networking.k8s.io/httproute-gw2-l1-i0 ["api-1.dnspolicy-scale-test-loadbalanced.mn.hcpapps.net"] 9m26s
113+
scale-test-1 httproute.gateway.networking.k8s.io/httproute-gw1-l1-i1 ["api-1.dnspolicy-scale-test-loadbalanced.mn.hcpapps.net"] 9m25s
114+
scale-test-1 httproute.gateway.networking.k8s.io/httproute-gw2-l1-i1 ["api-1.dnspolicy-scale-test-loadbalanced.mn.hcpapps.net"] 9m25s
115+
116+
NAMESPACE NAME TYPE DATA AGE
117+
scale-test-0 secret/aws-credentials kuadrant.io/aws 3 11m
118+
scale-test-1 secret/aws-credentials kuadrant.io/aws 3 11m
119+
120+
NAMESPACE NAME AGE
121+
scale-test-0 dnspolicy.kuadrant.io/dnspolicy-gw1-i0 9m26s
122+
scale-test-0 dnspolicy.kuadrant.io/dnspolicy-gw2-i0 9m26s
123+
scale-test-1 dnspolicy.kuadrant.io/dnspolicy-gw1-i1 9m25s
124+
scale-test-1 dnspolicy.kuadrant.io/dnspolicy-gw2-i1 9m25s
125+
```
126+
127+
Records Created in AWS:
128+
129+
![img.png](img.png)
130+
131+
## Useful commands
132+
133+
Delete all resources created by the test, useful if you used SKIP_CLEANUP=true
134+
```shell
135+
kubectl delete gateway,httproute,dnspolicy,all -A -l app=scale-test
136+
```
137+
138+
Tail all logs of dns operators created by a specific kubeburner job
139+
```shell
140+
kubectl stern -l kube-burner-job=dnspolicy-scale-test-setup-namespaced-dns-operators -A
141+
```

Diff for: ‎scale_test/dnspolicy/gw-dnspolicy-loadbalanced.yaml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
apiVersion: kuadrant.io/v1
2+
kind: DNSPolicy
3+
metadata:
4+
name: dnspolicy-gw{{ .GW_NUM }}-i{{ .Iteration }}
5+
labels:
6+
app: scale-test
7+
spec:
8+
targetRef:
9+
group: gateway.networking.k8s.io
10+
kind: Gateway
11+
name: gw{{ .GW_NUM }}-i{{ .Iteration }}
12+
providerRefs:
13+
- name: {{ .DNS_PROVIDER }}-credentials
14+
loadBalancing:
15+
weight: 120
16+
geo: {{ .GeoCode }}
17+
defaultGeo: true

Diff for: ‎scale_test/dnspolicy/gw.yaml

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
apiVersion: gateway.networking.k8s.io/v1
2+
kind: Gateway
3+
metadata:
4+
name: gw{{.GW_NUM}}-i{{.Iteration}}
5+
labels:
6+
app: scale-test
7+
spec:
8+
gatewayClassName: istio
9+
listeners:
10+
{{- $numListeners := .NUM_LISTENERS | atoi }}
11+
{{- range $index := until $numListeners }}
12+
{{- $listenerNum := add1 $index }}
13+
- allowedRoutes:
14+
namespaces:
15+
from: All
16+
hostname: api-{{ $listenerNum }}.{{ $.JobName }}.{{ $.KUADRANT_ZONE_ROOT_DOMAIN }}
17+
name: api-{{$listenerNum}}
18+
port: 80
19+
protocol: HTTP
20+
{{- end }}

Diff for: ‎scale_test/dnspolicy/httproute.yaml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
apiVersion: gateway.networking.k8s.io/v1
2+
kind: HTTPRoute
3+
metadata:
4+
name: httproute-gw{{ .GW_NUM }}-l{{ .LISTENER_NUM }}-i{{ .Iteration }}
5+
labels:
6+
app: scale-test
7+
spec:
8+
parentRefs:
9+
- group: gateway.networking.k8s.io
10+
kind: Gateway
11+
name: gw{{ .GW_NUM }}-i{{ .Iteration }}
12+
hostnames:
13+
- api-{{ .LISTENER_NUM }}.{{ .JobName }}.{{ .KUADRANT_ZONE_ROOT_DOMAIN }}
14+
rules:
15+
- backendRefs:
16+
- group: ''
17+
kind: Service
18+
name: httpbin
19+
port: 8080
20+
weight: 1
21+
matches:
22+
- path:
23+
type: PathPrefix
24+
value: /

Diff for: ‎scale_test/dnspolicy/img.png

164 KB
Loading

0 commit comments

Comments
 (0)