Skip to content

Commit 9c6f248

Browse files
author
Jakub Smolar
authored
Merge pull request #564 from Kuadrant/comparison-script
Add comparison script and readme updates
2 parents 4cc795d + 7d12747 commit 9c6f248

File tree

2 files changed

+350
-0
lines changed

2 files changed

+350
-0
lines changed

Diff for: ‎scale_test/compare.sh

+291
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,291 @@
1+
#!/usr/bin/env bash
2+
3+
# Elasticsearch parameters
4+
ES_INDEX="kube-burner"
5+
ES_URL="$ES_SERVER/$ES_INDEX/_search"
6+
7+
# If UUID1 and UUID2 are not set, get the most recent two jobSummary entries
8+
if [ -z "$UUID1" ] || [ -z "$UUID2" ]; then
9+
# Get the most recent two jobSummary entries
10+
curl -k -s -X GET "$ES_URL" \
11+
-H 'Content-Type: application/json' \
12+
-d '{
13+
"size": 2,
14+
"query": {
15+
"bool": {
16+
"must": [
17+
{ "term": { "metricName.keyword": "jobSummary" } },
18+
{ "term": { "jobConfig.name.keyword": "scale-test-main" } }
19+
]
20+
}
21+
},
22+
"sort": [
23+
{ "timestamp": { "order": "desc" } }
24+
],
25+
"_source": ["uuid", "elapsedTime", "timestamp"]
26+
}' > jobSummary.json
27+
28+
# Extract the uuids, elapsedTime values, and timestamps
29+
UUID1=$(jq -r '.hits.hits[1]._source.uuid' jobSummary.json)
30+
UUID2=$(jq -r '.hits.hits[0]._source.uuid' jobSummary.json)
31+
32+
ELAPSED1=$(jq -r '.hits.hits[1]._source.elapsedTime' jobSummary.json)
33+
ELAPSED2=$(jq -r '.hits.hits[0]._source.elapsedTime' jobSummary.json)
34+
35+
TIMESTAMP1=$(jq -r '.hits.hits[1]._source.timestamp' jobSummary.json)
36+
TIMESTAMP2=$(jq -r '.hits.hits[0]._source.timestamp' jobSummary.json)
37+
else
38+
# Use the provided UUID1 and UUID2 to get elapsedTime and timestamp
39+
# Fetch ELAPSED1 and TIMESTAMP1 for UUID1
40+
response1=$(curl -k -s -X GET "$ES_URL" \
41+
-H 'Content-Type: application/json' \
42+
-d "{
43+
\"size\": 1,
44+
\"query\": {
45+
\"bool\": {
46+
\"must\": [
47+
{ \"term\": { \"metricName.keyword\": \"jobSummary\" } },
48+
{ \"term\": { \"uuid.keyword\": \"$UUID1\" } },
49+
{ \"term\": { \"jobConfig.name.keyword\": \"scale-test-main\" } }
50+
]
51+
}
52+
},
53+
\"_source\": [\"elapsedTime\", \"timestamp\"]
54+
}")
55+
ELAPSED1=$(echo "$response1" | jq -r '.hits.hits[0]._source.elapsedTime')
56+
TIMESTAMP1=$(echo "$response1" | jq -r '.hits.hits[0]._source.timestamp')
57+
58+
# Fetch ELAPSED2 and TIMESTAMP2 for UUID2
59+
response2=$(curl -k -s -X GET "$ES_URL" \
60+
-H 'Content-Type: application/json' \
61+
-d "{
62+
\"size\": 1,
63+
\"query\": {
64+
\"bool\": {
65+
\"must\": [
66+
{ \"term\": { \"metricName.keyword\": \"jobSummary\" } },
67+
{ \"term\": { \"uuid.keyword\": \"$UUID2\" } },
68+
{ \"term\": { \"jobConfig.name.keyword\": \"scale-test-main\" } }
69+
]
70+
}
71+
},
72+
\"_source\": [\"elapsedTime\", \"timestamp\"]
73+
}")
74+
ELAPSED2=$(echo "$response2" | jq -r '.hits.hits[0]._source.elapsedTime')
75+
TIMESTAMP2=$(echo "$response2" | jq -r '.hits.hits[0]._source.timestamp')
76+
fi
77+
78+
# Shorten UUIDs for column headers
79+
UUID_SHORT1="${UUID1:0:8}"
80+
UUID_SHORT2="${UUID2:0:8}"
81+
82+
# Function to get per-namespace CPU values for a given uuid
83+
get_namespace_metric_values() {
84+
local uuid="$1"
85+
local metricName="$2"
86+
87+
curl -k -s -X GET "$ES_URL" \
88+
-H 'Content-Type: application/json' \
89+
-d "{
90+
\"size\": 0,
91+
\"query\": {
92+
\"bool\": {
93+
\"must\": [
94+
{ \"term\": { \"metricName.keyword\": \"$metricName\" } },
95+
{ \"term\": { \"uuid.keyword\": \"$uuid\" } },
96+
{ \"term\": { \"jobName.keyword\": \"scale-test-main\" } }
97+
]
98+
}
99+
},
100+
\"aggs\": {
101+
\"namespaces\": {
102+
\"terms\": {
103+
\"field\": \"labels.namespace.keyword\",
104+
\"size\": 1000
105+
},
106+
\"aggs\": {
107+
\"avg_value\": {
108+
\"avg\": {
109+
\"field\": \"value\"
110+
}
111+
}
112+
}
113+
}
114+
}
115+
}" | jq -r '.aggregations.namespaces.buckets[] | [ .key, (.avg_value.value | tostring) ] | @tsv'
116+
}
117+
118+
# Function to get per-controller reconcile time values for a given uuid
119+
get_controller_reconcile() {
120+
local uuid="$1"
121+
122+
curl -k -s -X GET "$ES_URL" \
123+
-H 'Content-Type: application/json' \
124+
-d "{
125+
\"size\": 50,
126+
\"query\": {
127+
\"bool\": {
128+
\"must\": [
129+
{ \"term\": { \"metricName.keyword\": \"Controller99thReconcile\" } },
130+
{ \"term\": { \"uuid.keyword\": \"$uuid\" } },
131+
{ \"term\": { \"jobName.keyword\": \"scale-test-main\" } }
132+
]
133+
}
134+
},
135+
\"_source\": [\"value\", \"labels.controller\"],
136+
\"sort\": [
137+
{ \"timestamp\": { \"order\": \"desc\" } }
138+
]
139+
}" | jq -r '.hits.hits[] | [ .["_source"]["labels"]["controller"], .["_source"]["value"] ] | @tsv'
140+
}
141+
142+
# Get per-namespace CPU values
143+
declare -A CPU1_VALUES
144+
declare -A CPU2_VALUES
145+
while IFS=$'\t' read -r namespace value; do
146+
if [ -z "${CPU1_VALUES["$namespace"]}" ]; then
147+
CPU1_VALUES["$namespace"]=$value
148+
fi
149+
done < <(get_namespace_metric_values "$UUID1" "namespaceCPU")
150+
151+
while IFS=$'\t' read -r namespace value; do
152+
if [ -z "${CPU2_VALUES["$namespace"]}" ]; then
153+
CPU2_VALUES["$namespace"]=$value
154+
fi
155+
done < <(get_namespace_metric_values "$UUID2" "namespaceCPU")
156+
157+
ALL_CPU_NAMESPACES=()
158+
for namespace in "${!CPU1_VALUES[@]}"; do
159+
ALL_CPU_NAMESPACES+=("$namespace")
160+
done
161+
for namespace in "${!CPU2_VALUES[@]}"; do
162+
if [[ ! " ${ALL_CPU_NAMESPACES[@]} " =~ " ${namespace} " ]]; then
163+
ALL_CPU_NAMESPACES+=("$namespace")
164+
fi
165+
done
166+
167+
# Get per-namespace Memory values
168+
declare -A MEM1_VALUES
169+
declare -A MEM2_VALUES
170+
while IFS=$'\t' read -r namespace value; do
171+
if [ -z "${MEM1_VALUES["$namespace"]}" ]; then
172+
MEM1_VALUES["$namespace"]=$value
173+
fi
174+
done < <(get_namespace_metric_values "$UUID1" "namespaceMemory")
175+
176+
while IFS=$'\t' read -r namespace value; do
177+
if [ -z "${MEM2_VALUES["$namespace"]}" ]; then
178+
MEM2_VALUES["$namespace"]=$value
179+
fi
180+
done < <(get_namespace_metric_values "$UUID2" "namespaceMemory")
181+
182+
ALL_MEM_NAMESPACES=()
183+
for namespace in "${!MEM1_VALUES[@]}"; do
184+
ALL_MEM_NAMESPACES+=("$namespace")
185+
done
186+
for namespace in "${!MEM2_VALUES[@]}"; do
187+
if [[ ! " ${ALL_MEM_NAMESPACES[@]} " =~ " ${namespace} " ]]; then
188+
ALL_MEM_NAMESPACES+=("$namespace")
189+
fi
190+
done
191+
192+
# Get per-controller reconcile time values
193+
declare -A RECONCILE1_VALUES
194+
declare -A RECONCILE2_VALUES
195+
while IFS=$'\t' read -r namespace value; do
196+
if [ -z "${RECONCILE1_VALUES["$namespace"]}" ]; then
197+
RECONCILE1_VALUES["$namespace"]=$value
198+
fi
199+
done < <(get_controller_reconcile "$UUID1")
200+
201+
while IFS=$'\t' read -r namespace value; do
202+
if [ -z "${RECONCILE2_VALUES["$namespace"]}" ]; then
203+
RECONCILE2_VALUES["$namespace"]=$value
204+
fi
205+
done < <(get_controller_reconcile "$UUID2")
206+
207+
ALL_RECONCILE_CONTROLLERS=()
208+
for namespace in "${!RECONCILE1_VALUES[@]}"; do
209+
ALL_RECONCILE_CONTROLLERS+=("$namespace")
210+
done
211+
for namespace in "${!RECONCILE2_VALUES[@]}"; do
212+
if [[ ! " ${ALL_RECONCILE_CONTROLLERS[@]} " =~ " ${namespace} " ]]; then
213+
ALL_RECONCILE_CONTROLLERS+=("$namespace")
214+
fi
215+
done
216+
217+
# Compare the values and compute differences
218+
compare_values() {
219+
local val1="$1"
220+
local val2="$2"
221+
local format="$3"
222+
local diff
223+
local sign
224+
225+
diff=$(echo "$val2 - $val1" | bc -l)
226+
if (( $(echo "$diff < 0" | bc -l) )); then
227+
sign="-"
228+
diff=$(echo "$val1 - $val2" | bc -l)
229+
else
230+
sign="+"
231+
fi
232+
233+
printf "%s${format}" "$sign" "$diff"
234+
}
235+
236+
ELAPSED_DIFF=$(compare_values "$ELAPSED1" "$ELAPSED2" "%.0f")
237+
238+
# Output the report
239+
printf "%-25s %-27s %-27s %-20s\n" "Metric" "UUID 1 ($UUID_SHORT1)" "UUID 2 ($UUID_SHORT2)" "Diff"
240+
printf "%-25s %-27s %-27s %-20s\n" "Timestamp" "$TIMESTAMP1" "$TIMESTAMP2" ""
241+
printf "%-25s %-27s %-27s %-20s\n" "Elapsed Time" "$ELAPSED1" "$ELAPSED2" "$ELAPSED_DIFF"
242+
printf "Namespace CPU\n"
243+
for namespace in "${ALL_CPU_NAMESPACES[@]}"; do
244+
value1="${CPU1_VALUES[$namespace]}"
245+
value2="${CPU2_VALUES[$namespace]}"
246+
247+
if [ -z "$value1" ]; then value1=0; fi
248+
if [ -z "$value2" ]; then value2=0; fi
249+
diff=$(compare_values "$value1" "$value2" "%.4f")
250+
251+
value1_formatted=$(printf "%.4f" "$value1")
252+
value2_formatted=$(printf "%.4f" "$value2")
253+
254+
printf "%-25s %-27s %-27s %-20s\n" "$namespace" "$value1_formatted" "$value2_formatted" "$diff"
255+
done
256+
printf "Namespace Memory (MB)\n"
257+
for namespace in "${ALL_MEM_NAMESPACES[@]}"; do
258+
value1="${MEM1_VALUES[$namespace]}"
259+
value2="${MEM2_VALUES[$namespace]}"
260+
261+
if [ -z "$value1" ]; then value1=0; fi
262+
if [ -z "$value2" ]; then value2=0; fi
263+
diff=$(echo "$value2 - $value1" | bc -l)
264+
sign=""
265+
if (( $(echo "$diff < 0" | bc -l) )); then
266+
sign="-"
267+
diff=$(echo "$value1 - $value2" | bc -l)
268+
else
269+
sign="+"
270+
fi
271+
diff=$(echo "scale=2; $diff / 1048576" | bc)
272+
273+
value1_formatted=$(echo "scale=2; $value1 / 1048576" | bc)
274+
value2_formatted=$(echo "scale=2; $value2 / 1048576" | bc)
275+
276+
printf "%-25s %-27s %-27s %s%-20s\n" "$namespace" "$value1_formatted" "$value2_formatted" "$sign" "$diff"
277+
done
278+
printf "Controller 99th Reconcile (s)\n"
279+
for namespace in "${ALL_RECONCILE_CONTROLLERS[@]}"; do
280+
value1="${RECONCILE1_VALUES[$namespace]}"
281+
value2="${RECONCILE2_VALUES[$namespace]}"
282+
283+
if [ -z "$value1" ]; then value1=0; fi
284+
if [ -z "$value2" ]; then value2=0; fi
285+
diff=$(compare_values "$value1" "$value2" "%.4f")
286+
287+
value1_formatted=$(printf "%.4f" "$value1")
288+
value2_formatted=$(printf "%.4f" "$value2")
289+
290+
printf "%-25s %-27s %-27s %-20s\n" "$namespace" "$value1_formatted" "$value2_formatted" "$diff"
291+
done

Diff for: ‎scale_test/readme.md

+59
Original file line numberDiff line numberDiff line change
@@ -179,3 +179,62 @@ Some common transforms are:
179179
* "Sort by"
180180

181181
Inspecting some of the existing panels in the Kuadrant dashboard and kube-burner dashboards at https://github.com/kube-burner/kube-burner/tree/main/examples/grafana-dashboards can give inspiration as well.
182+
183+
## Performance Comparison Script
184+
185+
The `./compare.sh` script allows you to compare key performance metrics between two test runs. It fetches metrics from an Elasticsearch instance and generates a report highlighting the differences in performance between the two specified runs, similar to the comparison Grafana dashboard. Values are calculated based on the average of most recent values.
186+
187+
You can specify the UUIDs of the two test runs you wish to compare by setting the `UUID1` and `UUID2` environment variables when running the script.
188+
189+
```bash
190+
UUID1=<first-test-uuid> UUID2=<second-test-uuid> ./compare.sh
191+
```
192+
193+
**Example**:
194+
195+
```bash
196+
UUID1=926c7847-95bd-4ed2-bf80-2cad6746db9c UUID2=c1195a3d-6ceb-4468-951c-d0492ef7c79c ./compare.sh
197+
```
198+
199+
### Automatic UUID Selection
200+
201+
If you do not specify `UUID1` and `UUID2`, the script will automatically fetch the two most recent test runs from Elasticsearch and compare them.
202+
203+
```bash
204+
./compare.sh
205+
```
206+
207+
## Example Output
208+
209+
```
210+
Metric UUID 1 (926c7847) UUID 2 (c1195a3d) Diff
211+
Timestamp 2024-10-18T12:01:11.588843Z 2024-10-18T12:50:47.638731Z
212+
Elapsed Time 152 220 +68
213+
214+
Namespace CPU (Average Values)
215+
Namespace 926c7847 c1195a3d Diff
216+
kuadrant-system 0.0119 0.0122 +0.0003
217+
scale-test-0 0.0006 0.1076 +0.1070
218+
gateway-system 0.0022 0.0019 -0.0003
219+
istio-system 0.0071 0.0317 +0.0246
220+
221+
Namespace Memory (MB)
222+
Namespace 926c7847 c1195a3d Diff
223+
kuadrant-system 181.78 152.12 -29.65
224+
scale-test-0 168.64 267.74 +99.09
225+
gateway-system 40.30 39.59 -0.70
226+
istio-system 114.27 119.07 +4.79
227+
228+
Controller 99th Reconcile (s)
229+
Controller 926c7847 c1195a3d Diff
230+
dnsrecord 5.8800 2.9600 -2.9200
231+
authpolicy 0.1835 0.0955 -0.0880
232+
dnspolicy 0.1410 0.1423 +0.0012
233+
kuadrant 0.0980 0.1475 +0.0495
234+
limitador 0.0980 0.3975 +0.2995
235+
gateway 0.1474 0.0885 -0.0588
236+
controller 3.4350 1.2150 -2.2200
237+
tlspolicy 3.8300 0.0959 -3.7341
238+
ratelimitpolicy 0.1760 0.0902 -0.0858
239+
authorino 0.0000 0.6955 +0.6955
240+
```

0 commit comments

Comments
 (0)