Uyuni Health Check Tool Disconnected Solution

health-check/.gitignore

@@ -0,0 +1,12 @@
+build
+dist
+.eggs
+*.egg-info
+logcli-linux-amd64
+promtail-linux-amd64
+__pycache__
+**/config/exporter/config.yaml
+**/config/promtail/config.yaml
+**/config/grafana/dashboards/supportconfig_with_logs.json
+
+.vscode/

health-check/README.md

@@ -0,0 +1,41 @@
+### uyuni-health-check
+
+A tool providing dashboard, metrics and logs from an Uyuni server supportconfig to visualise its health status.
+
+## Requirements
+
+* `python3`
+* `podman`
+
+## Building and installing
+
+Install the tool locally into a virtual environment:
+
+```
+python3 -m venv venv
+. venv/bin/activate
+pip install .
+```
+
+## Getting started
+
+This tool builds and deploys the necessary containers to scrape some metrics and logs from an Uyuni server supportconfig directory.
+Execute the `run` phase of the tool as such:
+
+```
+uyuni-health-check -s ~/path/to/supportconfig run --logs --from_datetime=2024-01-01T00:00:00Z --to_datetime=2024-06-01T20:00:00Z
+```
+
+This will create and start the following containers locally:
+
+- uyuni-health-exporter (port `9000`)
+- grafana (port `3000`)
+- loki (port `9100`)
+- promtail (port `9081`)
+
+After you start the containers, visit `localhost:3000` and select the `Supportconfig with Logs` dashboard.
+If necessary, the default username/password for Grafana is `admin:admin`.
+
+## Security notes
+After running this tool, and until containers are destroyed, the Grafana Dashboards (and other metrics) are exposing metrics and logs messages that may contain sensitive data and information to any non-root user in the system or to anyone that have access to this host in the network.
+

health-check/pyproject.toml

@@ -0,0 +1,43 @@
+# SPDX-FileCopyrightText: 2023 SUSE LLC
+#
+# SPDX-License-Identifier: Apache-2.0
+
+[project]
+name = "uyuni-health-check"
+description = "Show Uyuni server health metrics and logs"
+readme = "README.md"
+requires-python = ">=3.6"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "Click",
+    "rich",
+    "requests",
+    "Jinja2",
+    "PyYAML",
+]
+maintainers = [
+    {name = "Pablo Suárez Hernández", email = "[email protected]"},
+]
+dynamic = ["version"]
+
+[project.urls]
+homepage = "https://github.com/uyuni-project/uyuni"
+tracker =  "https://github.com/uyuni-project/uyuni/issues"
+
+[project.scripts]
+uyuni-health-check = "uyuni_health_check.main:main"
+
+[tool.setuptools]
+package-dir = {"" = "src"}
+
+[build-system]
+requires = [
+    "setuptools>=42",
+    "setuptools_scm[toml]",
+    "wheel",
+]
+build-backend = "setuptools.build_meta"
+

health-check/src/uyuni_health_check/config.ini

@@ -0,0 +1,11 @@
+[podman]
+network_name = health-check-network
+
+[loki]
+loki_container_name = uyuni_health_check_loki
+loki_port = 3100
+jobs = cobbler,postgresql,rhn,apache
+
+[logcli]
+logcli_container_name = uyuni_health_check_logcli
+logcli_image_name = logcli

health-check/src/uyuni_health_check/config/grafana/alerts.yaml

@@ -0,0 +1,92 @@
+apiVersion: 1
+groups:
+    - orgId: 1
+      name: alert-eval
+      folder: alerts
+      interval: 1m
+      rules:
+        - uid: ce6i8dhdhj400e
+          title: More worker threads than CPUs
+          condition: C
+          data:
+            - refId: A
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: infinity
+              model:
+                columns: []
+                datasource:
+                    type: yesoreyeram-infinity-datasource
+                    uid: infinity
+                filters: []
+                format: table
+                global_query_id: ""
+                hide: false
+                intervalMs: 1000
+                maxDataPoints: 43200
+                parser: backend
+                refId: A
+                root_selector: salt_configuration[name="worker_threads"].value
+                source: url
+                type: json
+                url: http://uyuni_health_check_supportconfig-exporter:9000/metrics.json
+                url_options:
+                    data: ""
+                    method: GET
+            - refId: B
+              relativeTimeRange:
+                from: 600
+                to: 0
+              datasourceUid: infinity
+              model:
+                columns: []
+                datasource:
+                    type: yesoreyeram-infinity-datasource
+                    uid: infinity
+                filters: []
+                format: table
+                global_query_id: ""
+                hide: false
+                intervalMs: 1000
+                maxDataPoints: 43200
+                parser: backend
+                refId: B
+                root_selector: hw[name="cpu_count"].value
+                source: url
+                type: json
+                url: http://uyuni_health_check_supportconfig-exporter:9000/metrics.json
+                url_options:
+                    data: ""
+                    method: GET
+            - refId: C
+              datasourceUid: __expr__
+              model:
+                conditions:
+                    - evaluator:
+                        params:
+                            - 0
+                            - 0
+                        type: gt
+                      operator:
+                        type: and
+                      query:
+                        params: []
+                      reducer:
+                        params: []
+                        type: avg
+                      type: query
+                datasource:
+                    name: Expression
+                    type: __expr__
+                    uid: __expr__
+                expression: $B - $A < 0
+                hide: false
+                intervalMs: 1000
+                maxDataPoints: 43200
+                refId: C
+                type: math
+          noDataState: NoData
+          execErrState: Error
+          for: 1m
+          isPaused: false

health-check/src/uyuni_health_check/config/grafana/dashboard.yaml

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2023 SUSE LLC
+#
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: 1
+
+providers:
+  - name: "Dashboard provider"
+    orgId: 1
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: false
+    options:
+      path: /var/lib/grafana/dashboards
+      foldersFromFilesStructure: true