Skip to content

Commit 82ed273

Browse files
benbzbenbz
committed
Make livenessProbes configurable
1 parent 84099ae commit 82ed273

31 files changed

+2000
-37
lines changed

‎charts/matrix-stack/source/common/probe.json

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
{
2+
"type": "object",
3+
"properties": {
4+
"failureThreshold": {
5+
"type": [
6+
"integer",
7+
"null"
8+
],
9+
"minimum": 1
10+
},
11+
"initialDelaySeconds": {
12+
"type": [
13+
"integer",
14+
"null"
15+
],
16+
"minimum": 0
17+
},
18+
"periodSeconds": {
19+
"type": [
20+
"integer",
21+
"null"
22+
],
23+
"minimum": 1
24+
},
25+
"successThreshold": {
26+
"type": [
27+
"integer",
28+
"null"
29+
],
30+
"minimum": 1
31+
},
32+
"timeoutSeconds": {
33+
"type": [
34+
"integer",
35+
"null"
36+
],
37+
"minimum": 1
38+
}
39+
}
40+
}

‎charts/matrix-stack/source/common/sub_schema_values.yaml.j2

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,53 @@ labels: {}
276276
{{ (credential("PostgreSQL password", "password", commented=True) | indent(2)) }}
277277
{%- endmacro %}
278278

279+
{% macro probe(type, failureThreshold=none, initialDelaySeconds=none, periodSeconds=none, successThreshold=none, timeoutSeconds=none) %}
280+
## Configuration of the thresholds and frequencies of the {{ type }}Probe
281+
{%- if failureThreshold is none and initialDelaySeconds is none and periodSeconds is none and successThreshold is none and timeoutSeconds is none %}
282+
# {{ type }}Probe:
283+
{%- else %}
284+
{{ type }}Probe:
285+
{%- endif %}
286+
## How many consecutive failures for the probe to be considered failed
287+
{%- if failureThreshold is not none %}
288+
failureThreshold: {{ failureThreshold }}
289+
{%- else %}
290+
# failureThreshold: 3
291+
{%- endif %}
292+
293+
## Number of seconds after the container has started before the probe starts
294+
{%- if initialDelaySeconds is not none %}
295+
initialDelaySeconds: {{ initialDelaySeconds }}
296+
{%- else %}
297+
# initialDelaySeconds: 0
298+
{%- endif %}
299+
300+
## How often (in seconds) to perform the probe
301+
{%- if periodSeconds is not none %}
302+
periodSeconds: {{ periodSeconds }}
303+
{%- else %}
304+
# periodSeconds: 1
305+
{%- endif %}
306+
307+
## How many consecutive successes for the probe to be consider successful after having failed
308+
{%- if successThreshold is not none %}
309+
{%- if type in ["startup", "liveness"] %}
310+
successThreshold: 1
311+
{%- else %}
312+
successThreshold: {{ successThreshold }}
313+
{%- endif %}
314+
{%- else %}
315+
# successThreshold: 1
316+
{%- endif %}
317+
318+
## Number of seconds after which the probe times out
319+
{%- if timeoutSeconds is not none %}
320+
timeoutSeconds: {{ timeoutSeconds }}
321+
{%- else %}
322+
# timeoutSeconds: 1
323+
{%- endif %}
324+
{%- endmacro %}
325+
279326
{% macro resources(requests_memory, requests_cpu, limits_memory, key='resources') %}
280327
## Kubernetes resources to allocate to each instance.
281328
{{ key }}:

‎charts/matrix-stack/source/element-web.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@
5151
},
5252
"topologySpreadConstraints": {
5353
"$ref": "file://common/topologySpreadConstraints.json"
54+
},
55+
"livenessProbe": {
56+
"$ref": "file://common/probe.json"
5457
}
5558
}
5659
}

‎charts/matrix-stack/source/element-web.yaml.j2

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,4 @@ replicas: 1
3030
{{- sub_schema_values.serviceAccount() -}}
3131
{{- sub_schema_values.tolerations() -}}
3232
{{- sub_schema_values.topologySpreadConstraints() }}
33+
{{- sub_schema_values.probe("liveness", failureThreshold=3, periodSeconds=10) }}

‎charts/matrix-stack/source/haproxy.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,9 @@
4242
},
4343
"topologySpreadConstraints": {
4444
"$ref": "file://common/topologySpreadConstraints.json"
45+
},
46+
"livenessProbe": {
47+
"$ref": "file://common/probe.json"
4548
}
4649
}
4750
}

‎charts/matrix-stack/source/haproxy.yaml.j2

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,4 @@ replicas: 1
1818
{{- sub_schema_values.serviceMonitors() }}
1919
{{- sub_schema_values.tolerations() }}
2020
{{- sub_schema_values.topologySpreadConstraints() }}
21+
{{- sub_schema_values.probe("liveness", initialDelaySeconds=10, timeoutSeconds=5) }}

‎charts/matrix-stack/source/matrix-rtc.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,9 @@
8686
"topologySpreadConstraints": {
8787
"$ref": "file://common/topologySpreadConstraints.json"
8888
},
89+
"livenessProbe": {
90+
"$ref": "file://common/probe.json"
91+
},
8992
"sfu": {
9093
"type": "object",
9194
"properties": {
@@ -173,6 +176,9 @@
173176
},
174177
"tolerations": {
175178
"$ref": "file://common/tolerations.json"
179+
},
180+
"livenessProbe": {
181+
"$ref": "file://common/probe.json"
176182
}
177183
}
178184
}

‎charts/matrix-stack/source/matrix-rtc.yaml.j2

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ enabled: true
3030
{{- sub_schema_values.serviceMonitors() }}
3131
{{- sub_schema_values.serviceAccount() }}
3232
{{- sub_schema_values.tolerations() }}
33+
{{- sub_schema_values.probe("liveness") }}
3334

3435
sfu:
3536
enabled: true
@@ -68,3 +69,4 @@ sfu:
6869
{{- sub_schema_values.serviceAccount() | indent(2) }}
6970
{{- sub_schema_values.serviceMonitors() | indent(2) }}
7071
{{- sub_schema_values.tolerations() | indent(2) }}
72+
{{- sub_schema_values.probe("liveness") | indent(2) }}

‎charts/matrix-stack/source/matrixAuthenticationService.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,9 @@
8585
},
8686
"topologySpreadConstraints": {
8787
"$ref": "file://common/topologySpreadConstraints.json"
88+
},
89+
"livenessProbe": {
90+
"$ref": "file://common/probe.json"
8891
}
8992
}
9093
}

‎charts/matrix-stack/source/matrixAuthenticationService.yaml.j2

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,3 +42,4 @@ privateKeys:
4242
{{ sub_schema_values.workloadAnnotations() }}
4343
{{ sub_schema_values.serviceMonitors() }}
4444
{{ sub_schema_values.extraEnv() }}
45+
{{ sub_schema_values.probe("liveness", failureThreshold=3) }}

‎charts/matrix-stack/source/postgres.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@
2323
},
2424
"resources": {
2525
"$ref": "file://common/resources.json"
26+
},
27+
"livenessProbe": {
28+
"$ref": "file://common/probe.json"
2629
}
2730
}
2831
},
@@ -73,6 +76,9 @@
7376
"topologySpreadConstraints": {
7477
"$ref": "file://common/topologySpreadConstraints.json"
7578
},
79+
"livenessProbe": {
80+
"$ref": "file://common/probe.json"
81+
},
7682
"serviceMonitors": {
7783
"$ref": "file://common/serviceMonitors.json"
7884
}

‎charts/matrix-stack/source/postgres.yaml.j2

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ postgresExporter:
99
{{- sub_schema_values.image(registry='docker.io', repository='prometheuscommunity/postgres-exporter', tag='v0.17.0') | indent(2) }}
1010
{{- sub_schema_values.resources(requests_memory='10Mi', requests_cpu='10m', limits_memory='500Mi')| indent(2) }}
1111
{{- sub_schema_values.containersSecurityContext()| indent(2) }}
12+
{{- sub_schema_values.probe("liveness", periodSeconds=6, timeoutSeconds=2) | indent(2) }}
1213

1314
{{- sub_schema_values.credential("Postgres Admin Password", "adminPassword", initIfAbsent=true) }}
1415

@@ -29,3 +30,4 @@ essPasswords:
2930
{{- sub_schema_values.serviceMonitors() }}
3031
{{- sub_schema_values.tolerations() }}
3132
{{- sub_schema_values.topologySpreadConstraints() }}
33+
{{- sub_schema_values.probe("liveness", initialDelaySeconds=45, timeoutSeconds=2) }}

‎charts/matrix-stack/source/synapse.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,9 @@
172172
"topologySpreadConstraints": {
173173
"$ref": "file://common/topologySpreadConstraints.json"
174174
},
175+
"livenessProbe": {
176+
"$ref": "file://common/probe.json"
177+
},
175178
"workers": {
176179
"type": "object",
177180
"properties": {
@@ -269,6 +272,9 @@
269272
},
270273
"tolerations": {
271274
"$ref": "file://common/tolerations.json"
275+
},
276+
"livenessProbe": {
277+
"$ref": "file://common/probe.json"
272278
}
273279
}
274280
}

‎charts/matrix-stack/source/synapse.yaml.j2

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ logging:
9696
{{- sub_schema_values.serviceMonitors() }}
9797
{{- sub_schema_values.tolerations() }}
9898
{{- sub_schema_values.topologySpreadConstraints() }}
99+
{{- sub_schema_values.probe("liveness", failureThreshold=8, periodSeconds=6, timeoutSeconds=2) }}
99100

100101
## Extra command line arguments to provide to Synapse
101102
extraArgs: []
@@ -110,3 +111,4 @@ redis:
110111
{{- sub_schema_values.resources(requests_memory='50Mi', requests_cpu='50m', limits_memory='50Mi') | indent(2) }}
111112
{{- sub_schema_values.serviceAccount() | indent(2) }}
112113
{{- sub_schema_values.tolerations() | indent(2) }}
114+
{{- sub_schema_values.probe("liveness", initialDelaySeconds=15) | indent(2) }}

‎charts/matrix-stack/source/synapse/scalable_worker.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
},
1616
"topologySpreadConstraints": {
1717
"$ref": "file://common/topologySpreadConstraints.json"
18+
},
19+
"livenessProbe": {
20+
"$ref": "file://common/probe.json"
1821
}
1922
},
2023
"type": "object"

‎charts/matrix-stack/source/synapse/single_worker.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
},
66
"resources": {
77
"$ref": "file://common/resources.json"
8+
},
9+
"livenessProbe": {
10+
"$ref": "file://common/probe.json"
811
}
912
},
1013
"type": "object"

‎charts/matrix-stack/source/synapse/synapse_sub_schema_values.yaml.j2

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ Copyright 2024 New Vector Ltd
44
SPDX-License-Identifier: AGPL-3.0-only
55
#}
66

7+
{% import 'sub_schema_values.yaml.j2' as sub_schema_values -%}
8+
79
{% macro single_worker(workerType) %}
810
{{ workerType }}:
911
## Set to true to deploy this worker
@@ -12,6 +14,8 @@ SPDX-License-Identifier: AGPL-3.0-only
1214
## Resources for this worker.
1315
## If omitted the global Synapse resources are used
1416
# resources: {}
17+
18+
{{- sub_schema_values.probe("liveness", failureThreshold=8, periodSeconds=6, timeoutSeconds=2) | indent(2) }}
1519
{%- endmacro %}
1620

1721
{% macro scalable_worker(workerType) %}
@@ -25,4 +29,6 @@ SPDX-License-Identifier: AGPL-3.0-only
2529
## Resources for this worker.
2630
## If omitted the global Synapse resources are used
2731
# resources: {}
32+
33+
{{- sub_schema_values.probe("liveness", failureThreshold=3, periodSeconds=6, timeoutSeconds=2) | indent(2) }}
2834
{%- endmacro %}

‎charts/matrix-stack/templates/element-web/deployment.yaml

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,13 +75,11 @@ spec:
7575
- containerPort: 8080
7676
name: element
7777
protocol: TCP
78-
livenessProbe:
79-
failureThreshold: 3
78+
livenessProbe: {{- include "element-io.ess-library.pods.probe" .livenessProbe | nindent 10 }}
8079
httpGet:
8180
path: /health
8281
port: element
8382
scheme: HTTP
84-
periodSeconds: 10
8583
readinessProbe:
8684
failureThreshold: 3
8785
httpGet:

‎charts/matrix-stack/templates/ess-library/_pods.tpl

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,3 +81,26 @@ topologySpreadConstraints:
8181
{{- end }}
8282
{{- end }}
8383
{{- end }}
84+
85+
{{- /*
86+
The only differences this has over (. | toYaml) are that
87+
* it allows us to omit nullable values for individual properties
88+
* it renders nothing rather than {} when no properties are specified.
89+
*/ -}}
90+
{{- define "element-io.ess-library.pods.probe" -}}
91+
{{- with .failureThreshold }}
92+
failureThreshold: {{ . }}
93+
{{- end }}
94+
{{- with .initialDelaySeconds }}
95+
initialDelaySeconds: {{ . }}
96+
{{- end }}
97+
{{- with .periodSeconds }}
98+
periodSeconds: {{ . }}
99+
{{- end }}
100+
{{- with .successThreshold }}
101+
successThreshold: {{ . }}
102+
{{- end }}
103+
{{- with .timeoutSeconds }}
104+
timeoutSeconds: {{ . }}
105+
{{- end }}
106+
{{- end }}

‎charts/matrix-stack/templates/haproxy/deployment.yaml

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,12 +122,10 @@ spec:
122122
{{- end }}
123123
periodSeconds: 2
124124
failureThreshold: 150
125-
livenessProbe:
125+
livenessProbe: {{- include "element-io.ess-library.pods.probe" .livenessProbe | nindent 10 }}
126126
httpGet:
127127
path: /haproxy_test
128128
port: haproxy-metrics
129-
initialDelaySeconds: 10
130-
timeoutSeconds: 5
131129
readinessProbe:
132130
httpGet:
133131
path: /haproxy_test

‎charts/matrix-stack/templates/matrix-authentication-service/deployment.yaml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -201,8 +201,7 @@ spec:
201201
port: internal
202202
scheme: HTTP
203203
initialDelaySeconds: 5
204-
livenessProbe:
205-
failureThreshold: 3
204+
livenessProbe: {{- include "element-io.ess-library.pods.probe" .livenessProbe | nindent 10 }}
206205
httpGet:
207206
path: /health
208207
port: internal

‎charts/matrix-stack/templates/matrix-rtc/sfu_deployment.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ spec:
171171
{{- end }}
172172
{{- end }}
173173
{{- end }}
174-
livenessProbe:
174+
livenessProbe: {{- include "element-io.ess-library.pods.probe" .livenessProbe | nindent 10 }}
175175
httpGet:
176176
path: /
177177
port: http

‎charts/matrix-stack/templates/matrix-rtc/sfu_jwt_deployment.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ spec:
6464
- containerPort: 8080
6565
name: http
6666
protocol: TCP
67-
livenessProbe:
67+
livenessProbe: {{- include "element-io.ess-library.pods.probe" .livenessProbe | nindent 10 }}
6868
httpGet:
6969
path: /healthz
7070
port: http

0 commit comments

Comments
 (0)