Make probe thresholds and frequencies configurable

charts/matrix-stack/source/common/probe.json

@@ -0,0 +1,40 @@
+{
+  "type": "object",
+  "properties": {
+    "failureThreshold": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 1
+    },
+    "initialDelaySeconds": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 0
+    },
+    "periodSeconds": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 1
+    },
+    "successThreshold": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 1
+    },
+    "timeoutSeconds": {
+      "type": [
+        "integer",
+        "null"
+      ],
+      "minimum": 1
+    }
+  }
+}

charts/matrix-stack/source/common/sub_schema_values.yaml.j2

@@ -276,6 +276,53 @@ labels: {}
 {{ (credential("PostgreSQL password", "password", commented=True) | indent(2)) }}
 {%- endmacro %}
 
+{% macro probe(type, failureThreshold=none, initialDelaySeconds=none, periodSeconds=none, successThreshold=none, timeoutSeconds=none) %}
+## Configuration of the thresholds and frequencies of the {{ type }}Probe
+{%- if failureThreshold is none and initialDelaySeconds is none and periodSeconds is none and successThreshold is none and timeoutSeconds is none %}
+# {{ type }}Probe:
+{%- else %}
+{{ type }}Probe:
+{%- endif %}
+  ## How many consecutive failures for the probe to be considered failed
+{%- if failureThreshold is not none %}
+  failureThreshold: {{ failureThreshold }}
+{%- else %}
+  # failureThreshold: 3
+{%- endif %}
+
+  ## Number of seconds after the container has started before the probe starts
+{%- if initialDelaySeconds is not none %}
+  initialDelaySeconds: {{ initialDelaySeconds }}
+{%- else %}
+  # initialDelaySeconds: 0
+{%- endif %}
+
+  ## How often (in seconds) to perform the probe
+{%- if periodSeconds is not none %}
+  periodSeconds: {{ periodSeconds }}
+{%- else %}
+  # periodSeconds: 1
+{%- endif %}
+
+  ## How many consecutive successes for the probe to be consider successful after having failed
+{%- if successThreshold is not none %}
+{%- if type in ["startup", "liveness"] %}
+  successThreshold: 1
+{%- else %}
+  successThreshold: {{ successThreshold }}
+{%- endif %}
+{%- else %}
+  # successThreshold: 1
+{%- endif %}
+
+  ## Number of seconds after which the probe times out
+{%- if timeoutSeconds is not none %}
+  timeoutSeconds: {{ timeoutSeconds }}
+{%- else %}
+  # timeoutSeconds: 1
+{%- endif %}
+{%- endmacro %}
+
 {% macro resources(requests_memory, requests_cpu, limits_memory, key='resources') %}
 ## Kubernetes resources to allocate to each instance.
 {{ key }}:

charts/matrix-stack/source/element-web.json

@@ -51,6 +51,15 @@
     },
     "topologySpreadConstraints": {
       "$ref": "file://common/topologySpreadConstraints.json"
+    },
+    "livenessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "readinessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "startupProbe": {
+      "$ref": "file://common/probe.json"
     }
   }
 }

charts/matrix-stack/source/element-web.yaml.j2

@@ -30,3 +30,6 @@ replicas: 1
 {{- sub_schema_values.serviceAccount() -}}
 {{- sub_schema_values.tolerations() -}}
 {{- sub_schema_values.topologySpreadConstraints() }}
+{{- sub_schema_values.probe("liveness", failureThreshold=3, periodSeconds=10) }}
+{{- sub_schema_values.probe("readiness", failureThreshold=3, periodSeconds=3) }}
+{{- sub_schema_values.probe("startup", failureThreshold=3, initialDelaySeconds=2, periodSeconds=3) }}

charts/matrix-stack/source/haproxy.json

@@ -42,6 +42,15 @@
     },
     "topologySpreadConstraints": {
       "$ref": "file://common/topologySpreadConstraints.json"
+    },
+    "livenessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "readinessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "startupProbe": {
+      "$ref": "file://common/probe.json"
     }
   }
 }

charts/matrix-stack/source/haproxy.yaml.j2

@@ -18,3 +18,8 @@ replicas: 1
 {{- sub_schema_values.serviceMonitors() }}
 {{- sub_schema_values.tolerations() }}
 {{- sub_schema_values.topologySpreadConstraints() }}
+{{- sub_schema_values.probe("liveness", initialDelaySeconds=10, timeoutSeconds=5) }}
+{{- sub_schema_values.probe("readiness", initialDelaySeconds=20, timeoutSeconds=5) }}
+# The failureThreshold here is tweaked towards Synapse being ready
+# If Synapse isn't being deployed, unsetting this or setting it to 3 maybe more appropriate
+{{- sub_schema_values.probe("startup", failureThreshold=150, periodSeconds=2) }}

charts/matrix-stack/source/matrix-rtc.json

@@ -86,6 +86,15 @@
     "topologySpreadConstraints": {
       "$ref": "file://common/topologySpreadConstraints.json"
     },
+    "livenessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "readinessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "startupProbe": {
+      "$ref": "file://common/probe.json"
+    },
     "sfu": {
       "type": "object",
       "properties": {
@@ -173,6 +182,15 @@
         },
         "tolerations": {
           "$ref": "file://common/tolerations.json"
+        },
+        "livenessProbe": {
+          "$ref": "file://common/probe.json"
+        },
+        "readinessProbe": {
+          "$ref": "file://common/probe.json"
+        },
+        "startupProbe": {
+          "$ref": "file://common/probe.json"
         }
       }
     }

charts/matrix-stack/source/matrix-rtc.yaml.j2

@@ -30,6 +30,9 @@ enabled: true
 {{- sub_schema_values.serviceMonitors() }}
 {{- sub_schema_values.serviceAccount() }}
 {{- sub_schema_values.tolerations() }}
+{{- sub_schema_values.probe("liveness") }}
+{{- sub_schema_values.probe("readiness") }}
+{{- sub_schema_values.probe("startup") }}
 
 sfu:
   enabled: true
@@ -68,3 +71,6 @@ sfu:
 {{- sub_schema_values.serviceAccount() | indent(2) }}
 {{- sub_schema_values.serviceMonitors() | indent(2) }}
 {{- sub_schema_values.tolerations() | indent(2) }}
+{{- sub_schema_values.probe("liveness") | indent(2) }}
+{{- sub_schema_values.probe("readiness") | indent(2) }}
+{{- sub_schema_values.probe("startup") | indent(2) }}

charts/matrix-stack/source/matrixAuthenticationService.json

@@ -85,6 +85,15 @@
     },
     "topologySpreadConstraints": {
       "$ref": "file://common/topologySpreadConstraints.json"
+    },
+    "livenessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "readinessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "startupProbe": {
+      "$ref": "file://common/probe.json"
     }
   }
 }

charts/matrix-stack/source/matrixAuthenticationService.yaml.j2

@@ -42,3 +42,6 @@ privateKeys:
 {{ sub_schema_values.workloadAnnotations() }}
 {{ sub_schema_values.serviceMonitors() }}
 {{ sub_schema_values.extraEnv() }}
+{{ sub_schema_values.probe("liveness", failureThreshold=3) }}
+{{ sub_schema_values.probe("readiness", failureThreshold=3) }}
+{{ sub_schema_values.probe("startup", initialDelaySeconds=5) }}

charts/matrix-stack/source/postgres.json

@@ -23,6 +23,15 @@
         },
         "resources": {
           "$ref": "file://common/resources.json"
+        },
+        "livenessProbe": {
+          "$ref": "file://common/probe.json"
+        },
+        "readinessProbe": {
+          "$ref": "file://common/probe.json"
+        },
+        "startupProbe": {
+          "$ref": "file://common/probe.json"
         }
       }
     },
@@ -73,6 +82,15 @@
     "topologySpreadConstraints": {
       "$ref": "file://common/topologySpreadConstraints.json"
     },
+    "livenessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "readinessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "startupProbe": {
+      "$ref": "file://common/probe.json"
+    },
     "serviceMonitors": {
       "$ref": "file://common/serviceMonitors.json"
     }

charts/matrix-stack/source/postgres.yaml.j2

@@ -9,6 +9,9 @@ postgresExporter:
   {{- sub_schema_values.image(registry='docker.io', repository='prometheuscommunity/postgres-exporter', tag='v0.17.0') | indent(2) }}
   {{- sub_schema_values.resources(requests_memory='10Mi', requests_cpu='10m', limits_memory='500Mi')| indent(2) }}
   {{- sub_schema_values.containersSecurityContext()| indent(2) }}
+  {{- sub_schema_values.probe("liveness", periodSeconds=6, timeoutSeconds=2) | indent(2) }}
+  {{- sub_schema_values.probe("readiness", periodSeconds=2, successThreshold=2, timeoutSeconds=2) | indent(2) }}
+  {{- sub_schema_values.probe("startup", failureThreshold=20, periodSeconds=2) | indent(2) }}
 
 {{- sub_schema_values.credential("Postgres Admin Password", "adminPassword", initIfAbsent=true) }}
 
@@ -29,3 +32,6 @@ essPasswords:
 {{- sub_schema_values.serviceMonitors() }}
 {{- sub_schema_values.tolerations() }}
 {{- sub_schema_values.topologySpreadConstraints() }}
+{{- sub_schema_values.probe("liveness", initialDelaySeconds=45, timeoutSeconds=2) }}
+{{- sub_schema_values.probe("readiness", initialDelaySeconds=15, timeoutSeconds=2) }}
+{{- sub_schema_values.probe("startup") }}

charts/matrix-stack/source/synapse.json

@@ -172,6 +172,15 @@
     "topologySpreadConstraints": {
       "$ref": "file://common/topologySpreadConstraints.json"
     },
+    "livenessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "readinessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "startupProbe": {
+      "$ref": "file://common/probe.json"
+    },
     "workers": {
       "type": "object",
       "properties": {
@@ -269,6 +278,15 @@
         },
         "tolerations": {
           "$ref": "file://common/tolerations.json"
+        },
+        "livenessProbe": {
+          "$ref": "file://common/probe.json"
+        },
+        "readinessProbe": {
+          "$ref": "file://common/probe.json"
+        },
+        "startupProbe": {
+          "$ref": "file://common/probe.json"
         }
       }
     }

charts/matrix-stack/source/synapse.yaml.j2

@@ -96,6 +96,9 @@ logging:
 {{- sub_schema_values.serviceMonitors() }}
 {{- sub_schema_values.tolerations() }}
 {{- sub_schema_values.topologySpreadConstraints() }}
+{{- sub_schema_values.probe("liveness", failureThreshold=8, periodSeconds=6, timeoutSeconds=2) }}
+{{- sub_schema_values.probe("readiness", failureThreshold=8, periodSeconds=2, successThreshold=2, timeoutSeconds=2) }}
+{{- sub_schema_values.probe("startup", failureThreshold=54, periodSeconds=2) }}
 
 ## Extra command line arguments to provide to Synapse
 extraArgs: []
@@ -110,3 +113,6 @@ redis:
 {{- sub_schema_values.resources(requests_memory='50Mi', requests_cpu='50m', limits_memory='50Mi') | indent(2) }}
 {{- sub_schema_values.serviceAccount() | indent(2) }}
 {{- sub_schema_values.tolerations() | indent(2) }}
+{{- sub_schema_values.probe("liveness", initialDelaySeconds=15) | indent(2) }}
+{{- sub_schema_values.probe("readiness", initialDelaySeconds=5) | indent(2) }}
+{{- sub_schema_values.probe("startup") | indent(2) }}

charts/matrix-stack/source/synapse/scalable_worker.json

@@ -15,6 +15,15 @@
     },
     "topologySpreadConstraints": {
       "$ref": "file://common/topologySpreadConstraints.json"
+    },
+    "livenessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "readinessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "startupProbe": {
+      "$ref": "file://common/probe.json"
     }
   },
   "type": "object"

charts/matrix-stack/source/synapse/single_worker.json

@@ -5,6 +5,15 @@
     },
     "resources": {
       "$ref": "file://common/resources.json"
+    },
+    "livenessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "readinessProbe": {
+      "$ref": "file://common/probe.json"
+    },
+    "startupProbe": {
+      "$ref": "file://common/probe.json"
     }
   },
   "type": "object"

charts/matrix-stack/source/synapse/synapse_sub_schema_values.yaml.j2

@@ -4,6 +4,8 @@ Copyright 2024 New Vector Ltd
 SPDX-License-Identifier: AGPL-3.0-only
 #}
 
+{% import 'sub_schema_values.yaml.j2' as sub_schema_values -%}
+
 {% macro single_worker(workerType) %}
 {{ workerType }}:
   ## Set to true to deploy this worker
@@ -12,6 +14,10 @@ SPDX-License-Identifier: AGPL-3.0-only
   ## Resources for this worker.
   ## If omitted the global Synapse resources are used
   # resources: {}
+
+{{- sub_schema_values.probe("liveness", failureThreshold=8, periodSeconds=6, timeoutSeconds=2) | indent(2) }}
+{{- sub_schema_values.probe("readiness", failureThreshold=8, periodSeconds=2, successThreshold=2, timeoutSeconds=2) | indent(2) }}
+{{- sub_schema_values.probe("startup", failureThreshold=54, periodSeconds=2) | indent(2) }}
 {%- endmacro %}
 
 {% macro scalable_worker(workerType) %}
@@ -25,4 +31,8 @@ SPDX-License-Identifier: AGPL-3.0-only
   ## Resources for this worker.
   ## If omitted the global Synapse resources are used
   # resources: {}
+
+{{- sub_schema_values.probe("liveness", failureThreshold=3, periodSeconds=6, timeoutSeconds=2) | indent(2) }}
+{{- sub_schema_values.probe("readiness", failureThreshold=3, periodSeconds=2, successThreshold=2, timeoutSeconds=2) | indent(2) }}
+{{- sub_schema_values.probe("startup", failureThreshold=21, periodSeconds=2) | indent(2) }}
 {%- endmacro %}