rancher
diff --git a/‎.github/renovate.json
+1-1 b/‎.github/renovate.json
+1-1
diff --git a/‎.github/scripts/branch-tags.sh
-55 b/‎.github/scripts/branch-tags.sh
-55
diff --git a/‎.github/workflows/prom-fed-ci.yaml ‎.github/workflows/ci.yaml
+15-4 b/‎.github/workflows/prom-fed-ci.yaml ‎.github/workflows/ci.yaml
+15-4
diff --git a/‎.github/workflows/e2e/package/Dockerfile
+2-2 b/‎.github/workflows/e2e/package/Dockerfile
+2-2
diff --git a/‎.github/workflows/e2e/scripts/cluster-args.sh
+2-44 b/‎.github/workflows/e2e/scripts/cluster-args.sh
+2-44
diff --git a/‎.github/workflows/e2e/scripts/create-projecthelmchart.sh
+1-1 b/‎.github/workflows/e2e/scripts/create-projecthelmchart.sh
+1-1
diff --git a/‎.github/workflows/e2e/scripts/generate-artifacts.sh
+9 b/‎.github/workflows/e2e/scripts/generate-artifacts.sh
+9
diff --git a/‎.github/workflows/e2e/scripts/install-federator.sh
+6-5 b/‎.github/workflows/e2e/scripts/install-federator.sh
+6-5
diff --git a/‎.github/workflows/e2e/scripts/install-monitoring.sh
+2-1 b/‎.github/workflows/e2e/scripts/install-monitoring.sh
+2-1
diff --git a/‎.github/workflows/e2e/scripts/validate-project-alertmanager.sh
+53-20 b/‎.github/workflows/e2e/scripts/validate-project-alertmanager.sh
+53-20
diff --git a/‎.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh
+43-17 b/‎.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh
+43-17
@@ -15,7 +15,7 @@
     "dockerfile",
     "github-actions",
     "helm-values",
-    "regex"
+    "custom.regex"
   ],
   "packageRules": [
     {
 
@@ -33,18 +33,29 @@ jobs:
         arch:
           - x64
           - arm64
-    runs-on : runs-on,image=ubuntu22-full-${{ matrix.arch }},runner=4cpu-linux-${{ matrix.arch }},run-id=${{ github.run_id }}
+    runs-on: ${{ github.repository == 'rancher/prometheus-federator' && format('runs-on,image=ubuntu22-full-{1},runner=4cpu-linux-{1},run-id={0}', github.run_id, matrix.arch) || 'ubuntu-latest' }}
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
       - name : Set up Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2
         with:
           go-version: '1.22'
+      - name: Check if yq is installed
+        id: check_yq
+        run: |
+          if ! command -v yq &> /dev/null; then
+            echo "yq not found, installing..."
+            echo "::set-output name=install_yq::true"
+          else
+            echo "yq is already installed"
+            echo "::set-output name=install_yq::false"
+          fi
       - name : Install YQ
+        if: steps.check_yq.outputs.install_yq == 'true'
         run: |
           sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${{ matrix.arch == 'x64' && 'amd64' || matrix.arch  }} -O /usr/bin/yq && sudo chmod +x /usr/bin/yq;
       - name : Install helm
-        uses: azure/setup-helm@v3
+        uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Run CI
 
@@ -1,9 +1,9 @@
-FROM registry.suse.com/bci/golang:1.20 AS helm
+FROM registry.suse.com/bci/golang:1.23 AS helm
 RUN zypper -n install git
 RUN git -C / clone --branch release-v3.9.0 --depth=1 https://github.com/rancher/helm
 RUN make -C /helm
 
-FROM registry.suse.com/bci/golang:1.20
+FROM registry.suse.com/bci/golang:1.23
 
 ARG ARCH=amd64
 ENV KUBECTL_VERSION v1.21.8
 
@@ -8,55 +8,13 @@ cd $(dirname $0)/../../../..
 
 case "${KUBERNETES_DISTRIBUTION_TYPE}" in
 "k3s")
-    cluster_args=""
-    kubernetes_version=$(kubectl version | grep "Server Version" | cut -d ' ' -f3)
-    case "${kubernetes_version}" in
-    v1.23.*)
-        embedded_helm_controller_fixed_version="v1.23.14"
-        if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
-            cluster_args="--set helmProjectOperator.helmController.enabled=false"
-        fi
-        ;;
-    v1.24.*)
-        embedded_helm_controller_fixed_version="v1.24.8"
-        if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
-            cluster_args="--set helmProjectOperator.helmController.enabled=false"
-        fi
-        ;;
-    v1.25.*)
-        embedded_helm_controller_fixed_version="v1.25.4"
-        if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
-            cluster_args="--set helmProjectOperator.helmController.enabled=false"
-        fi
-        ;;
-    esac
+    cluster_args="--set helmProjectOperator.helmController.enabled=false"
     ;;
 "rke")
     cluster_args=""
     ;;
 "rke2")
-    cluster_args=""
-    kubernetes_version=$(kubectl version | grep "Server Version" | cut -d ' ' -f3)
-    case "${kubernetes_version}" in
-    v1.23.*)
-        embedded_helm_controller_fixed_version="v1.23.14"
-        if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
-            cluster_args="--set helmProjectOperator.helmController.enabled=false"
-        fi
-        ;;
-    v1.24.*)
-        embedded_helm_controller_fixed_version="v1.24.8"
-        if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
-            cluster_args="--set helmProjectOperator.helmController.enabled=false"
-        fi
-        ;;
-    v1.25.*)
-        embedded_helm_controller_fixed_version="v1.25.4"
-        if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then
-            cluster_args="--set helmProjectOperator.helmController.enabled=false"
-        fi
-        ;;
-    esac
+    cluster_args="--set helmProjectOperator.helmController.enabled=false"
     ;;
 *)
     echo "KUBERNETES_DISTRIBUTION_TYPE=${KUBERNETES_DISTRIBUTION_TYPE} is unknown"
 
@@ -11,7 +11,7 @@ if [[ "${E2E_CI}" == "true" ]]; then
 else
     kubectl apply -f ./examples/prometheus-federator/project-helm-chart.yaml
 fi
-sleep ${DEFAULT_SLEEP_TIMEOUT_SECONDS};
+sleep "${DEFAULT_SLEEP_TIMEOUT_SECONDS}";
 
 if ! kubectl get -n cattle-monitoring-system job/helm-install-cattle-project-p-example-monitoring; then
     echo "ERROR: Helm Install Job for Project Monitoring Stack was never created after ${DEFAULT_SLEEP_TIMEOUT_SECONDS} seconds"
 
@@ -34,6 +34,7 @@ case "${KUBERNETES_DISTRIBUTION_TYPE}" in
 esac
 
 ARTIFACT_DIRECTORY=artifacts
+DESCRIBE_DIRECTORY=${ARTIFACT_DIRECTORY}/described
 MANIFEST_DIRECTORY=${ARTIFACT_DIRECTORY}/manifests
 LOG_DIRECTORY=${ARTIFACT_DIRECTORY}/logs
 
@@ -114,3 +115,11 @@ kubectl logs deployment/cattle-project-p-example-monitoring-grafana -n cattle-pr
 kubectl logs deployment/cattle-project-p-example-monitoring-grafana -n cattle-project-p-example -c grafana-sc-dashboard > ${LOG_DIRECTORY}/project-monitoring/grafana_sc_dashboard.log || true
 kubectl logs deployment/cattle-project-p-example-monitoring-grafana -n cattle-project-p-example -c grafana-sc-datasources > ${LOG_DIRECTORY}/project-monitoring/grafana_sc_datasources.log || true
 kubectl logs deployment/cattle-project-p-example-monitoring-grafana -n cattle-project-p-example -c grafana-init-sc-datasources > ${LOG_DIRECTORY}/project-monitoring/grafana_init_sc_datasources.log || true
+
+# Resource Descriptions
+
+mkdir -p ${DESCRIBE_DIRECTORY}
+
+## Additional Context
+kubectl describe jobs -n cattle-monitoring-system helm-install-cattle-project-p-example-monitoring > ${DESCRIBE_DIRECTORY}/project-monitoring-helm-install-job.log
+kubectl describe pods -n cattle-monitoring-system -l job-name=helm-install-cattle-project-p-example-monitoring > ${DESCRIBE_DIRECTORY}/project-monitoring-helm-install-pod.log
@@ -8,11 +8,12 @@ source $(dirname $0)/cluster-args.sh
 cd $(dirname $0)/../../../..
 source "$(pwd)/scripts/util-team-charts"
 
-NEWEST_CHART_VERSION=$(newest-chart-version "prometheus-federator")
-fetch-team-chart "prometheus-federator" "$NEWEST_CHART_VERSION"
-LATEST_CHART_PATH="./build/charts/prometheus-federator-${NEWEST_CHART_VERSION}.tgz"
-tar -xvzf "$LATEST_CHART_PATH" -C ./build/charts/
+make package-helm
 
-helm upgrade --install --create-namespace -n cattle-monitoring-system prometheus-federator --set helmProjectOperator.image.repository=${REPO:-rancher}/prometheus-federator --set helmProjectOperator.image.tag=${TAG:-dev} ${cluster_args} ${RANCHER_HELM_ARGS} ./build/charts/prometheus-federator
+helm upgrade --install --create-namespace -n cattle-monitoring-system prometheus-federator \
+  --set helmProjectOperator.image.repository=${REPO:-rancher}/prometheus-federator \
+  --set helmProjectOperator.image.tag=${TAG:-dev} \
+  ${cluster_args} \
+  ${RANCHER_HELM_ARGS} ./build/charts/prometheus-federator
 
 echo "PASS: Prometheus Federator has been installed"
@@ -5,12 +5,13 @@ set -x
 source $(dirname $0)/entry
 
 HELM_REPO="rancher-charts"
+HELM_REPO_URL="https://charts.rancher.io"
 
 cd $(dirname $0)/../../../..
 
 helm version
 
-helm repo add ${HELM_REPO} https://charts.rancher.io
+helm repo add ${HELM_REPO} $HELM_REPO_URL
 helm repo update
 
 echo "Create required \`cattle-fleet-system\` namespace"
 
@@ -10,27 +10,60 @@ tmp_alerts_yaml=$(mktemp)
 trap 'cleanup' EXIT
 cleanup() {
     set +e
-    rm ${tmp_alerts_yaml}
+    rm "${tmp_alerts_yaml}"
 }
 
-if [[ -z "${RANCHER_TOKEN}" ]]; then
-    curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts | yq -P - > ${tmp_alerts_yaml}
-else
-    curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_alerts_yaml}
-fi
-
-if [[ $(yq '. | length' "${tmp_alerts_yaml}") != "1" ]]; then
-    echo "ERROR: Found the wrong number of alerts in Project Alertmanager, expected only 'Watchdog'"
-    cat ${tmp_alerts_yaml}
-    exit 1
-fi
-
-if [[ $(yq '.[0].labels.alertname' "${tmp_alerts_yaml}") != "Watchdog" ]]; then
-    echo "ERROR: Expected the only alert to be triggered on the Project Alertmanager to be 'Watchdog'"
-    cat ${tmp_alerts_yaml}
-    exit 1
-fi
-
-cat ${tmp_alerts_yaml}
+checkData() {
+  if [[ -z "${RANCHER_TOKEN}" ]]; then
+      curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts | yq -P - > "${tmp_alerts_yaml}"
+  else
+      curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > "${tmp_alerts_yaml}"
+  fi
+}
+
+WAIT_TIMEOUT="${KUBECTL_WAIT_TIMEOUT%s}"
+START_TIME=$(date +%s)
+while true; do
+  checkData
+  CHECKS_PASSED=0
+
+  # Check if timeout has been reached
+  CURRENT_TIME=$(date +%s)
+  ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+  if [[ $ELAPSED_TIME -ge $WAIT_TIMEOUT ]]; then
+      echo "ERROR: Timeout reached, condition not met."
+      exit 1
+  fi
+
+  ALERT_COUNT=$(yq '. | length' "${tmp_alerts_yaml}")
+  if [[ $ALERT_COUNT -gt 3 ]]; then
+      echo "ERROR: Found too many alerts in Project Alertmanager. Expected at most: 'Watchdog', 'InfoInhibitor' and/or 'PrometheusOutOfOrderTimestamps'."
+      cat "${tmp_alerts_yaml}"
+
+      echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+      sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+      continue
+  fi
+  CHECKS_PASSED=$((CHECKS_PASSED+1))
+
+  UNEXPECTED_COUNT=$(yq '[.[] | select(.labels.alertname != "Watchdog" and .labels.alertname != "InfoInhibitor" and .labels.alertname != "PrometheusOutOfOrderTimestamps")] | length' "${tmp_alerts_yaml}")
+  if [[ $UNEXPECTED_COUNT -gt 0 ]]; then
+    echo "ERROR: Unexpected alert(s) found in active alerts list."
+    cat "${tmp_alerts_yaml}"
+
+    echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+    sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+    continue
+  fi
+  CHECKS_PASSED=$((CHECKS_PASSED+1))
+
+  if [[ $CHECKS_PASSED -eq 2 ]];then
+    # Get final elapsed time
+    ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+    break
+  fi
+done
+
+cat "${tmp_alerts_yaml}"
 
 echo "PASS: Project Alertmanager is up and running"
@@ -13,11 +13,13 @@ cleanup() {
     rm ${tmp_dashboards_yaml}
 }
 
-if [[ -z "${RANCHER_TOKEN}" ]]; then
-    curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search | yq -P - > ${tmp_dashboards_yaml}
-else
-    curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_dashboards_yaml}
-fi
+checkData() {
+  if [[ -z "${RANCHER_TOKEN}" ]]; then
+      curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search | yq -P - > ${tmp_dashboards_yaml}
+  else
+      curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_dashboards_yaml}
+  fi
+}
 
 expected_dashboards=(
     db/alertmanager-overview
@@ -41,18 +43,42 @@ expected_dashboards=(
     db/rancher-workload-pods
 );
 
-if [[ $(yq '.[].uri' ${tmp_dashboards_yaml} | wc -l | xargs) != "${#expected_dashboards[@]}" ]]; then
-    echo "ERROR: Found the wrong number of dashboards in Project Grafana, expected only the following: ${expected_dashboards[@]}"
-    cat ${tmp_dashboards_yaml}
-    exit 1
-fi      
-
-for dashboard in "${expected_dashboards[@]}"; do
-    if ! yq '.[].uri' ${tmp_dashboards_yaml} | grep "${dashboard}" 1>/dev/null; then
-        echo "ERROR: Expected '${dashboard}' to exist amongst ${#expected_dashboards[@]} dashboards in Project Grafana"
-        cat ${tmp_dashboards_yaml}
-        exit 1
-    fi
+WAIT_TIMEOUT="${KUBECTL_WAIT_TIMEOUT%s}"
+START_TIME=$(date +%s)
+while true; do
+  checkData
+
+  # Check if timeout has been reached
+  CURRENT_TIME=$(date +%s)
+  ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+  if [[ $ELAPSED_TIME -ge $WAIT_TIMEOUT ]]; then
+      echo "ERROR: Timeout reached, condition not met."
+      exit 1
+  fi
+
+  if [[ $(yq '.[].uri' ${tmp_dashboards_yaml} | wc -l | xargs) != "${#expected_dashboards[@]}" ]]; then
+    echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+    sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+    continue
+  fi
+
+  FOUND_DASHBOARDS=0
+  for dashboard in "${expected_dashboards[@]}"; do
+      if ! yq '.[].uri' ${tmp_dashboards_yaml} | grep "${dashboard}" 1>/dev/null; then
+          echo "ERROR: Expected '${dashboard}' to exist amongst ${#expected_dashboards[@]} dashboards in Project Grafana"
+          cat ${tmp_dashboards_yaml}
+          echo "Retrying in $DEFAULT_SLEEP_TIMEOUT_SECONDS seconds..."
+          sleep "$DEFAULT_SLEEP_TIMEOUT_SECONDS"
+          break
+      fi
+      FOUND_DASHBOARDS=$((FOUND_DASHBOARDS+1))
+  done
+
+  if [[ FOUND_DASHBOARDS -eq 19 ]];then
+    # Get final elapsed time
+    ELAPSED_TIME=$((CURRENT_TIME - START_TIME))
+    break
+  fi
 done
 
 cat ${tmp_dashboards_yaml}
Original file line number	Diff line number	Diff line change
`@@ -15,7 +15,7 @@`
`15`	`15`	`"dockerfile",`
`16`	`16`	`"github-actions",`
`17`	`17`	`"helm-values",`
`18`		`- "regex"`
	`18`	`+ "custom.regex"`
`19`	`19`	`],`
`20`	`20`	`"packageRules": [`
`21`	`21`	`{`