Skip to content

Commit

Permalink
fixup: Move promtail readiness check to after nodes have started
Browse files Browse the repository at this point in the history
  • Loading branch information
maru-ava committed Feb 24, 2025
1 parent f56bbca commit 4dce5a1
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 18 deletions.
14 changes: 10 additions & 4 deletions tests/fixture/e2e/env.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,11 @@ func (te *TestEnvironment) Marshal() []byte {
func NewTestEnvironment(tc tests.TestContext, flagVars *FlagVars, desiredNetwork *tmpnet.Network) *TestEnvironment {
require := require.New(tc)

// Start collectors for any command but stop
if flagVars.StartCollectors() && !flagVars.StopNetwork() {
require.NoError(tmpnet.StartCollectors(tc.DefaultContext(), tc.Log()))
}

var network *tmpnet.Network
// Need to load the network if it is being stopped or reused
if flagVars.StopNetwork() || flagVars.ReuseNetwork() {
Expand Down Expand Up @@ -130,10 +135,6 @@ func NewTestEnvironment(tc tests.TestContext, flagVars *FlagVars, desiredNetwork
}
}

if flagVars.StartCollectors() {
require.NoError(tmpnet.StartCollectors(tc.DefaultContext(), tc.Log()))
}

// Start a new network
if network == nil {
network = desiredNetwork
Expand All @@ -151,6 +152,11 @@ func NewTestEnvironment(tc tests.TestContext, flagVars *FlagVars, desiredNetwork
)
}

// Once one or more nodes are running it should be safe to wait for promtail to report readiness
if flagVars.StartCollectors() {
require.NoError(tmpnet.WaitForPromtailReadiness(tc.DefaultContext(), tc.Log()))
}

if flagVars.StartNetwork() {
os.Exit(0)
}
Expand Down
25 changes: 11 additions & 14 deletions tests/fixture/tmpnet/monitor_processes.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,23 +63,22 @@ func StartCollectors(ctx context.Context, log logging.Logger) error {
return err
}

// Wait for readiness. These checks are performed separately from start to
// minimize time to readiness.
readinessURLs := map[string]string{
promtailCmd: promtailReadinessURL,
prometheusCmd: prometheusReadinessURL,
}
for cmdName, readinessURLs := range readinessURLs {
if err := waitForReadiness(ctx, log, cmdName, readinessURLs); err != nil {
return err
}
log.Info("skipping promtail readiness check until one or more nodes have written their service discovery configuration")
if err := waitForReadiness(ctx, log, prometheusCmd, prometheusReadinessURL); err != nil {
return err
}

log.Info("To stop: tmpnetctl stop-collectors")

return nil
}

// WaitForPromtailReadiness waits until prometheus is ready. It can only succeed after
// one or more nodes have written their service discovery configuration.
func WaitForPromtailReadiness(ctx context.Context, log logging.Logger) error {
return waitForReadiness(ctx, log, promtailCmd, promtailReadinessURL)
}

// EnsureCollectorsStopped ensures collectors are not running.
func StopCollectors(ctx context.Context, log logging.Logger) error {
if _, ok := ctx.Deadline(); !ok {
Expand Down Expand Up @@ -354,7 +353,7 @@ func getPrometheusURL() string {
}

func getLokiURL() string {
return GetEnvWithDefault("LOKI_URL", defaultPrometheusURL)
return GetEnvWithDefault("LOKI_URL", defaultLokiURL)
}

// getCollectorCredentials retrieves the username and password for the command.
Expand Down Expand Up @@ -488,9 +487,7 @@ func waitForReadiness(ctx context.Context, log logging.Logger, cmdName string, r
}); err != nil {
return err
}
log.Info(cmdName+" ready",
zap.String("url", readinessURL),
)
log.Info(cmdName + " ready")
return nil
}

Expand Down

0 comments on commit 4dce5a1

Please sign in to comment.