Adjust Alertmanager E2E verification to allow common alerts (#190)

* Adjust Alertmanager E2E verification to allow InfoInhibitor & PrometheusOutOfOrderTimestamps

* Make helm repo url configurable

* Simplify checks to verify based on count vs (count - expected)

Author: mallardduck

Diff for: .github/workflows/e2e/scripts/install-monitoring.sh

@@ -5,12 +5,13 @@ set -x
 source $(dirname $0)/entry
 
 HELM_REPO="rancher-charts"
+HELM_REPO_URL="https://charts.rancher.io"
 
 cd $(dirname $0)/../../../..
 
 helm version
 
-helm repo add ${HELM_REPO} https://charts.rancher.io
+helm repo add ${HELM_REPO} $HELM_REPO_URL
 helm repo update
 
 echo "Create required \`cattle-fleet-system\` namespace"

Diff for: .github/workflows/e2e/scripts/validate-project-alertmanager.sh

@@ -35,24 +35,25 @@ while true; do
       exit 1
   fi
 
-  if [[ $(yq '. | length' "${tmp_alerts_yaml}") != "1" ]]; then
-      echo "ERROR: Found the wrong number of alerts in Project Alertmanager, expected only 'Watchdog'"
+  ALERT_COUNT=$(yq '. | length' "${tmp_alerts_yaml}")
+  if [[ $ALERT_COUNT -gt 3 ]]; then
+      echo "ERROR: Found too many alerts in Project Alertmanager. Expected at most: 'Watchdog', 'InfoInhibitor' and/or 'PrometheusOutOfOrderTimestamps'."
       cat "${tmp_alerts_yaml}"
 
-    echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
-    sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
-    continue
+      echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+      sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+      continue
   fi
   CHECKS_PASSED=$((CHECKS_PASSED+1))
 
+  UNEXPECTED_COUNT=$(yq '[.[] | select(.labels.alertname != "Watchdog" and .labels.alertname != "InfoInhibitor" and .labels.alertname != "PrometheusOutOfOrderTimestamps")] | length' "${tmp_alerts_yaml}")
+  if [[ $UNEXPECTED_COUNT -gt 0 ]]; then
+    echo "ERROR: Unexpected alert(s) found in active alerts list."
+    cat "${tmp_alerts_yaml}"
 
-  if [[ $(yq '.[0].labels.alertname' "${tmp_alerts_yaml}") != "Watchdog" ]]; then
-      echo "ERROR: Expected the only alert to be triggered on the Project Alertmanager to be 'Watchdog'"
-      cat "${tmp_alerts_yaml}"
-
-      echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
-      sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
-      continue
+    echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+    sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+    continue
   fi
   CHECKS_PASSED=$((CHECKS_PASSED+1))