-
Notifications
You must be signed in to change notification settings - Fork 721
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[tmpnet] Add check for metrics collection to monitoring action
- Loading branch information
Showing
6 changed files
with
189 additions
and
12 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| // Copyright (C) 2019-2024, Ava Labs, Inc. All rights reserved. | ||
| // See the file LICENSE for licensing terms. | ||
|
|
||
| package tmpnet | ||
|
|
||
| import ( | ||
| "context" | ||
| "errors" | ||
| "fmt" | ||
| "net/http" | ||
| "strings" | ||
| "time" | ||
|
|
||
| "github.com/prometheus/client_golang/api" | ||
| "github.com/prometheus/client_golang/api/prometheus/v1" | ||
| "github.com/prometheus/common/model" | ||
| "go.uber.org/zap" | ||
|
|
||
| "github.com/ava-labs/avalanchego/utils/logging" | ||
| ) | ||
|
|
||
| // CheckMetricsExist checks if metrics exist for the given | ||
| // network. Github labels are also included if provided as env vars | ||
| // (GH_*). | ||
| func CheckMetricsExist( | ||
| log logging.Logger, | ||
| networkUUID string, | ||
| ) error { | ||
| username, password, err := getCollectorCredentials(prometheusCmd) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to get collector credentials: %w", err) | ||
| } | ||
| query, err := getCheckMetricsQuery(networkUUID) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| url := getPrometheusURL() | ||
|
|
||
| log.Info("checking if metrics exist", | ||
| zap.String("url", url), | ||
| zap.String("query", query), | ||
| ) | ||
|
|
||
| results, err := queryPrometheus(log, url, username, password, query) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| metricsCount := len(results) | ||
| if metricsCount > 0 { | ||
| log.Info("metrics exist", | ||
| zap.String("query", query), | ||
| zap.Int("count", metricsCount), | ||
| ) | ||
| return nil | ||
| } | ||
|
|
||
| return errors.New("metrics not found") | ||
| } | ||
|
|
||
| // getCheckMetricsQuery returns the query to check if metrics exist. | ||
| func getCheckMetricsQuery(networkUUID string) (string, error) { | ||
| selectors := []string{} | ||
| if len(networkUUID) > 0 { | ||
| selectors = append(selectors, fmt.Sprintf("network_uuid=\"%s\"", networkUUID)) | ||
| } | ||
| githubLabels := githubLabelsFromEnv() | ||
| for label := range githubLabels { | ||
| value, err := githubLabels.GetStringVal(label) | ||
| if err != nil { | ||
| return "", err | ||
| } | ||
| if len(value) == 0 { | ||
| continue | ||
| } | ||
| selectors = append(selectors, fmt.Sprintf("%s=\"%s\"", label, value)) | ||
| } | ||
| return fmt.Sprintf("{%s}", strings.Join(selectors, ",")), nil | ||
| } | ||
|
|
||
| func queryPrometheus( | ||
| log logging.Logger, | ||
| url string, | ||
| username string, | ||
| password string, | ||
| query string, | ||
| ) (model.Vector, error) { | ||
| // Create client with basic auth | ||
| client, err := api.NewClient(api.Config{ | ||
| Address: url, | ||
| RoundTripper: &basicAuthRoundTripper{ | ||
| username: username, | ||
| password: password, | ||
| rt: api.DefaultRoundTripper, | ||
| }, | ||
| }) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to create client: %w", err) | ||
| } | ||
|
|
||
| // Query Prometheus | ||
| result, warnings, err := v1.NewAPI(client).Query( | ||
| context.Background(), | ||
| query, | ||
| time.Now(), | ||
| ) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("query failed: %w", err) | ||
| } | ||
| if len(warnings) > 0 { | ||
| log.Warn("prometheus query warnings", | ||
| zap.Strings("warnings", warnings), | ||
| ) | ||
| } | ||
|
|
||
| // Check results | ||
| vector, ok := result.(model.Vector) | ||
| if !ok { | ||
| return nil, fmt.Errorf("unexpected result type: %s", result.Type()) | ||
| } | ||
| return vector, nil | ||
| } | ||
|
|
||
| type basicAuthRoundTripper struct { | ||
| username, password string | ||
| rt http.RoundTripper | ||
| } | ||
|
|
||
| func (b *basicAuthRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { | ||
| req.SetBasicAuth(b.username, b.password) | ||
| return b.rt.RoundTrip(req) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters