Skip to content

Commit 023421e

Browse files
m3dwardsm3dwards
authored
Merge pull request #594 from pinheadmz/grafana-default
Deploy default grafana dashboard automatically
2 parents 46ad555 + d6473fe commit 023421e

File tree

10 files changed

+99
-70
lines changed

10 files changed

+99
-70
lines changed

‎docs/logging_monitoring.md

+8-21
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,11 @@ aggregate logs and data from Bitcoin RPC queries into a web-based dashboard.
6262
## Connect to logging dashboard
6363

6464
The logging stack including the user interface web server runs inside the kubernetes cluster.
65-
To access that from a local web browser, you must use kubernetes port-forwarding.
65+
Warnet will forward port `2019` locally from the cluster, and the landing page for all
66+
web based interfaces will be available at `localhost:2019`.
67+
68+
This page can also be opened quickly with the command [`warnet dashboard`](/docs/warnet.md#warnet-dashboard)
6669

67-
Run the script `./resources/scripts/connect_logging.sh` to forward port 3000.
68-
The Grafana dashboard will then be available locally at `localhost:3000`.
6970

7071
### Prometheus
7172

@@ -122,21 +123,7 @@ web-based interface.
122123
123124
#### Dashboards
124125
125-
To view the default metrics in the included default dashboard, upload the dashboard
126-
JSON file to the Grafana server:
127-
128-
```sh
129-
curl localhost:3000/api/dashboards/db \
130-
-H "Content-Type: application/json" \
131-
--data "{\"dashboard\": $(cat resources/configs/grafana/default_dashboard.json)}"
132-
```
133-
134-
Note the URL in the reply from the server (example):
135-
136-
```sh
137-
{"folderUid":"","id":2,"slug":"default-warnet-dashboard","status":"success","uid":"fdu0pda1z6a68b","url":"/d/fdu0pda1z6a68b/default-warnet-dashboard","version":1}(
138-
```
139-
140-
Open the dashboard in your browser (example):
141-
142-
`http://localhost:3000/d/fdu0pda1z6a68b/default-warnet-dashboard`
126+
Grafana dashboards are described in JSON files. A default Warnet dashboard
127+
is included and any other json files in the `/resources/charts/grafana-dashboards/files` directory
128+
will also be deployed to the web server. The Grafana UI itself also has an API
129+
for creating, exporting, and importing other dashboard files.

‎resources/charts/grafana-dashboards/.helmignore

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Patterns to ignore when building packages.
2+
# This supports shell glob matching, relative path matching, and
3+
# negation (prefixed with !). Only one pattern per line.
4+
.DS_Store
5+
# Common VCS dirs
6+
.git/
7+
.gitignore
8+
.bzr/
9+
.bzrignore
10+
.hg/
11+
.hgignore
12+
.svn/
13+
# Common backup files
14+
*.swp
15+
*.bak
16+
*.tmp
17+
*.orig
18+
*~
19+
# Various IDEs
20+
.project
21+
.idea/
22+
*.tmproj
23+
.vscode/

‎resources/charts/grafana-dashboards/Chart.yaml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
apiVersion: v2
2+
name: grafana-dashboards
3+
description: A Helm chart for Kubernetes
4+
5+
# A chart can be either an 'application' or a 'library' chart.
6+
#
7+
# Application charts are a collection of templates that can be packaged into versioned archives
8+
# to be deployed.
9+
#
10+
# Library charts provide useful utilities or functions for the chart developer. They're included as
11+
# a dependency of application charts to inject those utilities and functions into the rendering
12+
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
13+
type: application
14+
15+
# This is the chart version. This version number should be incremented each time you make changes
16+
# to the chart and its templates, including the app version.
17+
# Versions are expected to follow Semantic Versioning (https://semver.org/)
18+
version: 0.1.0
19+
20+
# This is the version number of the application being deployed. This version number should be
21+
# incremented each time you make changes to the application. Versions are not expected to
22+
# follow Semantic Versioning. They should reflect the version the application is using.
23+
# It is recommended to use it with quotes.
24+
appVersion: "1.16.0"

‎resources/configs/grafana/default_dashboard.json renamed to ‎resources/charts/grafana-dashboards/files/default_dashboard.json

+8-4
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@
2424
],
2525
"title": "Outbound connections",
2626
"type": "timeseries",
27-
"gridPos": {
27+
"gridPos":
28+
{
2829
"h": 8,
2930
"w": 24,
3031
"x": 0,
@@ -52,7 +53,8 @@
5253
],
5354
"title": "Inbound connections",
5455
"type": "timeseries",
55-
"gridPos": {
56+
"gridPos":
57+
{
5658
"h": 8,
5759
"w": 24,
5860
"x": 0,
@@ -80,7 +82,8 @@
8082
],
8183
"title": "Mempool size",
8284
"type": "timeseries",
83-
"gridPos": {
85+
"gridPos":
86+
{
8487
"h": 8,
8588
"w": 24,
8689
"x": 0,
@@ -108,7 +111,8 @@
108111
],
109112
"title": "Blocks",
110113
"type": "timeseries",
111-
"gridPos": {
114+
"gridPos":
115+
{
112116
"h": 8,
113117
"w": 24,
114118
"x": 0,

‎resources/charts/grafana-dashboards/templates/NOTES.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Grafana dashboards deployed

‎resources/charts/grafana-dashboards/templates/configmap.yaml

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: grafana-dashboards-config
5+
data:
6+
{{- $files := .Files.Glob "files/*.json" }}
7+
{{- range $path, $file := $files }}
8+
{{ base $path }}: |-
9+
{{ $file | toString | indent 4 }}
10+
{{- end }}

‎resources/manifests/grafana_values.yaml

+19
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,25 @@ datasources:
1212
- name: Loki
1313
type: loki
1414
url: http://loki-gateway.warnet-logging:80
15+
dashboardProviders:
16+
dashboardproviders.yaml:
17+
apiVersion: 1
18+
providers:
19+
- name: 'default'
20+
orgId: 1
21+
folder: ''
22+
type: file
23+
disableDeletion: false
24+
editable: true
25+
options:
26+
path: /var/lib/grafana/dashboards/default
27+
extraVolumeMounts:
28+
- name: grafana-dashboards-volume
29+
mountPath: /var/lib/grafana/dashboards/default
30+
extraVolumes:
31+
- name: grafana-dashboards-volume
32+
configMap:
33+
name: grafana-dashboards-config
1534
grafana.ini:
1635
auth:
1736
disable_login_form: true

‎resources/scripts/connect_logging.sh

-17
This file was deleted.

‎src/warnet/constants.py

+1
Original file line numberDiff line numberDiff line change
@@ -95,5 +95,6 @@
9595
f"helm upgrade --install --namespace warnet-logging --create-namespace --values {MANIFESTS_DIR}/loki_values.yaml loki grafana/loki --version 5.47.2",
9696
"helm upgrade --install --namespace warnet-logging promtail grafana/promtail",
9797
"helm upgrade --install --namespace warnet-logging prometheus prometheus-community/kube-prometheus-stack --namespace warnet-logging --set grafana.enabled=false",
98+
f"helm upgrade --install grafana-dashboards {CHARTS_DIR}/grafana-dashboards --namespace warnet-logging",
9899
f"helm upgrade --install --namespace warnet-logging loki-grafana grafana/grafana --values {MANIFESTS_DIR}/grafana_values.yaml",
99100
]

‎test/logging_test.py

+5-28
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,8 @@
11
#!/usr/bin/env python3
22

3-
import logging
43
import os
5-
import threading
64
from datetime import datetime
75
from pathlib import Path
8-
from subprocess import PIPE, Popen
96

107
import requests
118
from test_base import TestBase
@@ -18,39 +15,15 @@ def __init__(self):
1815
super().__init__()
1916
self.network_dir = Path(os.path.dirname(__file__)) / "data" / "logging"
2017
self.scripts_dir = Path(os.path.dirname(__file__)) / ".." / "resources" / "scripts"
21-
self.connect_logging_process = None
22-
self.connect_logging_thread = None
23-
self.connect_logging_logger = logging.getLogger("cnct_log")
2418

2519
def run_test(self):
2620
try:
2721
self.setup_network()
28-
self.start_logging()
22+
self.wait_for_endpoint_ready()
2923
self.test_prometheus_and_grafana()
3024
finally:
31-
if self.connect_logging_process is not None:
32-
self.log.info("Terminating background connect_logging.sh process...")
33-
self.connect_logging_process.terminate()
3425
self.cleanup()
3526

36-
def start_logging(self):
37-
# Stays alive in background
38-
self.connect_logging_process = Popen(
39-
[f"{self.scripts_dir / 'connect_logging.sh'}"],
40-
stdout=PIPE,
41-
stderr=PIPE,
42-
bufsize=1,
43-
universal_newlines=True,
44-
)
45-
self.log.info("connect_logging.sh started...")
46-
self.connect_logging_thread = threading.Thread(
47-
target=self.output_reader,
48-
args=(self.connect_logging_process.stdout, self.connect_logging_logger.info),
49-
)
50-
self.connect_logging_thread.daemon = True
51-
self.connect_logging_thread.start()
52-
self.wait_for_endpoint_ready()
53-
5427
def setup_network(self):
5528
self.log.info("Setting up network")
5629
self.log.info(self.warnet(f"deploy {self.network_dir}"))
@@ -118,6 +91,10 @@ def get_five_values_for_metric(metric):
11891
self.wait_for_predicate(lambda: get_five_values_for_metric("blocks"))
11992
self.wait_for_predicate(lambda: get_five_values_for_metric("txrate"))
12093

94+
# Verify default dashboard exists
95+
dbs = requests.get(f"{GRAFANA_URL}api/search").json()
96+
assert dbs[0]["title"] == "Default Warnet Dashboard"
97+
12198

12299
if __name__ == "__main__":
123100
test = LoggingTest()

0 commit comments

Comments
 (0)