Skip to content

Commit 9836c73

Browse files
bendanzhentanbendanzhentan
authored
add metrics (#32)
* feat: expose metrics via prometheus * feat: update docker-compose files * feat: change the metrics port * fix: metrics last scanned block
1 parent 017c6ac commit 9836c73

File tree

6 files changed

+188
-20
lines changed

6 files changed

+188
-20
lines changed

‎cmd/bot/run.go

+15-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"errors"
77
"fmt"
88
"math/big"
9+
"net/http"
910
"strings"
1011
"time"
1112

@@ -16,6 +17,7 @@ import (
1617
"github.com/ethereum/go-ethereum/common"
1718
"github.com/ethereum/go-ethereum/core/types"
1819
"github.com/ethereum/go-ethereum/log"
20+
"github.com/prometheus/client_golang/prometheus/promhttp"
1921
"github.com/urfave/cli/v2"
2022
"gorm.io/driver/mysql"
2123
"gorm.io/gorm"
@@ -61,10 +63,21 @@ func RunCommand(ctx *cli.Context) error {
6163

6264
l2ScannedBlock, err := queryL2ScannedBlock(db, cfg.L2StartingNumber)
6365
if err != nil {
64-
return err
66+
return fmt.Errorf("failed to query l2_scanned_blocks: %w", err)
67+
} else {
68+
logger.Info("starting from block", "blockNumber", l2ScannedBlock.Number)
6569
}
66-
logger.Info("starting from block", "blockNumber", l2ScannedBlock.Number)
6770

71+
go func() {
72+
http.Handle("/metrics", promhttp.Handler())
73+
http.Handle("/debug/metrics/prometheus", promhttp.Handler())
74+
err := http.ListenAndServe(":6060", nil)
75+
if err != nil {
76+
logger.Error("failed to start prometheus server", "error", err)
77+
}
78+
}()
79+
80+
go core.StartMetrics(ctx.Context, &cfg, &l1Client.Client, db, logger)
6881
go WatchBotDelegatedWithdrawals(ctx.Context, logger, db, l2Client, l2ScannedBlock, cfg)
6982
go ProcessBotDelegatedWithdrawals(ctx.Context, logger, db, l1Client, l2Client, cfg)
7083

‎core/metrics.go

+100
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package core
2+
3+
import (
4+
"context"
5+
"errors"
6+
"github.com/ethereum/go-ethereum/ethclient"
7+
"github.com/ethereum/go-ethereum/log"
8+
"github.com/prometheus/client_golang/prometheus"
9+
"github.com/prometheus/client_golang/prometheus/promauto"
10+
"gorm.io/gorm"
11+
"time"
12+
)
13+
14+
var (
15+
TxSignerBalance = promauto.NewGauge(prometheus.GaugeOpts{
16+
Name: "opbnb_bridge_bot_tx_signer_balance",
17+
Help: "The balance of the tx signer",
18+
})
19+
20+
ScannedBlockNumber = promauto.NewGauge(prometheus.GaugeOpts{
21+
Name: "opbnb_bridge_bot_scanned_block_number",
22+
Help: "The block number that has been scanned",
23+
})
24+
25+
UnprovenWithdrawals = promauto.NewGauge(prometheus.GaugeOpts{
26+
Name: "opbnb_bridge_bot_unproven_withdrawals",
27+
Help: "The number of unproven withdrawals",
28+
})
29+
UnfinalizedWithdrawals = promauto.NewGauge(prometheus.GaugeOpts{
30+
Name: "opbnb_bridge_bot_unfinalized_withdrawals",
31+
Help: "The number of unfinalized withdrawals",
32+
})
33+
EarliestUnProvenWithdrawalBlockNumber = promauto.NewGauge(prometheus.GaugeOpts{
34+
Name: "opbnb_bridge_bot_earliest_unproven_withdrawal_block_number",
35+
Help: "The earliest block number of unproven withdrawals",
36+
})
37+
EarliestUnfinalizedWithdrawalBlockNumber = promauto.NewGauge(prometheus.GaugeOpts{
38+
Name: "opbnb_bridge_bot_earliest_unfinalized_withdrawal_block_number",
39+
Help: "The earliest block number of unfinalized withdrawals",
40+
})
41+
42+
FailedWithdrawals = promauto.NewGauge(prometheus.GaugeOpts{
43+
Name: "opbnb_bridge_bot_failed_withdrawals",
44+
Help: "The number of failed withdrawals",
45+
})
46+
)
47+
48+
func StartMetrics(ctx context.Context, cfg *Config, l1Client *ethclient.Client, db *gorm.DB, logger log.Logger) {
49+
ticker := time.NewTicker(5 * time.Second)
50+
for range ticker.C {
51+
_, signerAddress, _ := cfg.SignerKeyPair()
52+
balance, err := l1Client.BalanceAt(ctx, *signerAddress, nil)
53+
if err != nil {
54+
logger.Error("failed to get signer balance", "error", err)
55+
}
56+
TxSignerBalance.Set(float64(balance.Int64()))
57+
58+
var scannedBlock L2ScannedBlock
59+
result := db.Last(&scannedBlock)
60+
if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) {
61+
logger.Error("failed to query scanned block", "error", result.Error)
62+
}
63+
ScannedBlockNumber.Set(float64(scannedBlock.Number))
64+
65+
var unprovenCnt int64
66+
result = db.Table("bot_delegated_withdrawals").Where("proven_time IS NULL AND failure_reason IS NULL").Count(&unprovenCnt)
67+
if result.Error != nil {
68+
logger.Error("failed to count withdrawals", "error", result.Error)
69+
}
70+
UnprovenWithdrawals.Set(float64(unprovenCnt))
71+
72+
var unfinalizedCnt int64
73+
result = db.Table("bot_delegated_withdrawals").Where("finalized_time IS NULL AND proven_time IS NOT NULL AND failure_reason IS NULL").Count(&unfinalizedCnt)
74+
if result.Error != nil {
75+
logger.Error("failed to count withdrawals", "error", result.Error)
76+
}
77+
UnfinalizedWithdrawals.Set(float64(unfinalizedCnt))
78+
79+
var failedCnt int64
80+
result = db.Table("bot_delegated_withdrawals").Where("failure_reason IS NOT NULL").Count(&failedCnt)
81+
if result.Error != nil {
82+
logger.Error("failed to count withdrawals", "error", result.Error)
83+
}
84+
FailedWithdrawals.Set(float64(failedCnt))
85+
86+
firstUnproven := BotDelegatedWithdrawal{}
87+
result = db.Table("bot_delegated_withdrawals").Order("id asc").Where("proven_time IS NULL AND failure_reason IS NULL").First(&firstUnproven)
88+
if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) {
89+
logger.Error("failed to query withdrawals", "error", result.Error)
90+
}
91+
EarliestUnProvenWithdrawalBlockNumber.Set(float64(firstUnproven.InitiatedBlockNumber))
92+
93+
firstUnfinalized := BotDelegatedWithdrawal{}
94+
result = db.Table("bot_delegated_withdrawals").Order("id asc").Where("finalized_time IS NULL AND proven_time IS NOT NULL AND failure_reason IS NULL").First(&firstUnfinalized)
95+
if result.Error != nil && !errors.Is(result.Error, gorm.ErrRecordNotFound) {
96+
logger.Error("failed to query withdrawals", "error", result.Error)
97+
}
98+
EarliestUnfinalizedWithdrawalBlockNumber.Set(float64(firstUnfinalized.InitiatedBlockNumber))
99+
}
100+
}

‎docker-compose.yml

-18
This file was deleted.

‎docker/docker-compose.yml

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
version: '3.8'
2+
3+
services:
4+
mysql:
5+
image: mysql:latest
6+
environment:
7+
- MYSQL_ROOT_PASSWORD=db_password
8+
- MYSQL_DATABASE=db_name
9+
- MYSQL_USER=db_username
10+
- MYSQL_PASSWORD=db_password
11+
command: --default-authentication-plugin=mysql_native_password
12+
ports:
13+
- "3306:3306"
14+
volumes:
15+
- mysql_data:/var/lib/mysql
16+
17+
prometheus:
18+
image: prom/prometheus
19+
container_name: prometheus
20+
command:
21+
- '--config.file=/etc/prometheus/prometheus.yml'
22+
ports:
23+
- 9090:9090
24+
restart: unless-stopped
25+
volumes:
26+
- ./prometheus:/etc/prometheus
27+
- prom_data:/prometheus
28+
29+
grafana:
30+
image: grafana/grafana
31+
container_name: grafana
32+
ports:
33+
- 3000:3000
34+
restart: unless-stopped
35+
environment:
36+
- GF_SECURITY_ADMIN_USER=admin
37+
- GF_SECURITY_ADMIN_PASSWORD=grafana
38+
volumes:
39+
- ./grafana:/etc/grafana/provisioning/datasources
40+
41+
volumes:
42+
mysql_data:
43+
prom_data:

‎docker/grafana/datasource.yml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
apiVersion: 1
2+
3+
datasources:
4+
- name: Prometheus
5+
type: prometheus
6+
url: http://prometheus:9090
7+
isDefault: true
8+
access: proxy
9+
editable: true

‎docker/prometheus/prometheus.yml

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
global:
2+
scrape_interval: 15s
3+
scrape_timeout: 10s
4+
evaluation_interval: 15s
5+
alerting:
6+
alertmanagers:
7+
- static_configs:
8+
- targets: []
9+
scheme: http
10+
timeout: 10s
11+
api_version: v1
12+
scrape_configs:
13+
- job_name: prometheus
14+
honor_timestamps: true
15+
scrape_interval: 15s
16+
scrape_timeout: 10s
17+
metrics_path: /metrics
18+
scheme: http
19+
static_configs:
20+
- targets:
21+
- localhost:9090

0 commit comments

Comments
 (0)