Use WaitUntilReady retry strategy for test pool

base/gocb_utils.go

@@ -193,8 +193,14 @@ func MgmtRequest(client *http.Client, mgmtEp, method, uri, contentType, username
 	return respBytes, response.StatusCode, nil
 }
 
+// CouchbaseClusterWaitUntilReadyOptions defines options when calling gocbcore.Agent.WaitUntilReady.
+type CouchbaseClusterWaitUntilReadyOptions struct {
+	RetryStrategy gocbcore.RetryStrategy // Retry strategy to use when waiting for the cluster to be ready
+	Timeout       time.Duration          // Timeout for waiting for the cluster to be ready
+}
+
 // NewClusterAgent creates a new gocbcore agent for a couchbase cluster.
-func NewClusterAgent(ctx context.Context, spec CouchbaseClusterSpec, waitUntilReadyTimeout time.Duration) (*gocbcore.Agent, error) {
+func NewClusterAgent(ctx context.Context, spec CouchbaseClusterSpec, waitUntilReadyOptions CouchbaseClusterWaitUntilReadyOptions) (*gocbcore.Agent, error) {
 	authenticator, err := GoCBCoreAuthConfig(spec.Username, spec.Password, spec.X509Certpath, spec.X509Keypath)
 	if err != nil {
 		return nil, err
@@ -226,7 +232,7 @@ func NewClusterAgent(ctx context.Context, spec CouchbaseClusterSpec, waitUntilRe
 
 	agent, err := gocbcore.CreateAgent(&config)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("gocbcore.CreateAgent failed: %w", err)
 	}
 
 	shouldCloseAgent := true
@@ -240,25 +246,26 @@ func NewClusterAgent(ctx context.Context, spec CouchbaseClusterSpec, waitUntilRe
 
 	agentReadyErr := make(chan error)
 	_, err = agent.WaitUntilReady(
-		time.Now().Add(waitUntilReadyTimeout),
+		time.Now().Add(waitUntilReadyOptions.Timeout),
 		gocbcore.WaitUntilReadyOptions{
-			ServiceTypes: []gocbcore.ServiceType{gocbcore.MgmtService},
+			ServiceTypes:  []gocbcore.ServiceType{gocbcore.MgmtService},
+			RetryStrategy: waitUntilReadyOptions.RetryStrategy,
 		},
 		func(result *gocbcore.WaitUntilReadyResult, err error) {
 			agentReadyErr <- err
 		},
 	)
 
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("gocbcore.Agent.WaitUntilReady failed: %w", err)
 	}
 
 	if err := <-agentReadyErr; err != nil {
 		if _, ok := errors.Unwrap(err).(x509.UnknownAuthorityError); ok {
 			err = fmt.Errorf("%w - Provide a CA cert, or set tls_skip_verify to true in config", err)
 		}
 
-		return nil, err
+		return nil, fmt.Errorf("cluster agent is not ready after %vs: %w", waitUntilReadyOptions.Timeout.Seconds(), err)
 	}
 
 	shouldCloseAgent = false

base/main_test_cluster.go

@@ -39,7 +39,11 @@ type tbpCluster struct {
 
 // newTestCluster returns a cluster based on the driver used by the defaultBucketSpec.
 func newTestCluster(ctx context.Context, clusterSpec CouchbaseClusterSpec) (*tbpCluster, error) {
-	agent, err := NewClusterAgent(ctx, clusterSpec, TestClusterReadyTimeout)
+	agent, err := NewClusterAgent(ctx, clusterSpec,
+		CouchbaseClusterWaitUntilReadyOptions{
+			Timeout:       TestClusterReadyTimeout,
+			RetryStrategy: gocbcore.NewBestEffortRetryStrategy(nil),
+		})
 	if err != nil {
 		return nil, fmt.Errorf("couldn't create cluster agent: %w", err)
 	}

rest/server_context.go

@@ -1841,8 +1841,6 @@ func (sc *ServerContext) updateCalculatedStats(ctx context.Context) {
 // Uses retry loop
 func (sc *ServerContext) initializeGoCBAgent(ctx context.Context) (*gocbcore.Agent, error) {
 	err, agent := base.RetryLoop(ctx, "Initialize Cluster Agent", func() (shouldRetry bool, err error, agent *gocbcore.Agent) {
-		// this timeout is pretty short since we are doing external retries
-		waitUntilReadyTimeout := 5 * time.Second
 		agent, err = base.NewClusterAgent(
 			ctx,
 			base.CouchbaseClusterSpec{
@@ -1853,7 +1851,11 @@ func (sc *ServerContext) initializeGoCBAgent(ctx context.Context) (*gocbcore.Age
 				X509Keypath:   sc.Config.Bootstrap.X509KeyPath,
 				CACertpath:    sc.Config.Bootstrap.CACertPath,
 				TLSSkipVerify: base.ValDefault(sc.Config.Bootstrap.ServerTLSSkipVerify, false),
-			}, waitUntilReadyTimeout)
+			},
+			base.CouchbaseClusterWaitUntilReadyOptions{
+				// this timeout is pretty short since we are doing external retries
+				Timeout: 5 * time.Second,
+			})
 		if err != nil {
 			// since we're starting up - let's be verbose (on console) about these retries happening ... otherwise it looks like nothing is happening ...
 			base.ConsolefCtx(ctx, base.LevelInfo, base.KeyConfig, "Couldn't initialize cluster agent: %v - will retry...", err)

rest/x509_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/couchbase/gocbcore/v10"
 	"github.com/couchbase/sync_gateway/base"
 	"github.com/couchbase/sync_gateway/db"
 	"github.com/stretchr/testify/assert"
@@ -88,7 +89,10 @@ func TestX509UnknownAuthorityWrap(t *testing.T) {
 			CACertpath:    sc.Bootstrap.CACertPath,
 			TLSSkipVerify: base.ValDefault(sc.Bootstrap.ServerTLSSkipVerify, false),
 		},
-		base.TestClusterReadyTimeout,
+		base.CouchbaseClusterWaitUntilReadyOptions{
+			Timeout:       base.TestClusterReadyTimeout,
+			RetryStrategy: gocbcore.NewBestEffortRetryStrategy(nil),
+		},
 	)
 	assert.Error(t, err)