*: Add configurable store timeout

Add a command line option to set the store timeout.

Author: Max

cmd/common.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sorintlab/stolon/internal/cluster"
@@ -50,12 +51,14 @@ type CommonConfig struct {
 	KubeConfig           string
 	KubeContext          string
 	KubeNamespace        string
+	StoreTimeout         time.Duration
 }
 
 func AddCommonFlags(cmd *cobra.Command, cfg *CommonConfig) {
 	cmd.PersistentFlags().StringVar(&cfg.ClusterName, "cluster-name", "", "cluster name")
 	cmd.PersistentFlags().StringVar(&cfg.StoreBackend, "store-backend", "", "store backend type (etcdv2/etcd, etcdv3, consul or kubernetes)")
 	cmd.PersistentFlags().StringVar(&cfg.StoreEndpoints, "store-endpoints", "", "a comma-delimited list of store endpoints (use https scheme for tls communication) (defaults: http://127.0.0.1:2379 for etcd, http://127.0.0.1:8500 for consul)")
+	cmd.PersistentFlags().DurationVar(&cfg.StoreTimeout, "store-timeout", cluster.DefaultStoreTimeout, "store request timeout")
 	cmd.PersistentFlags().StringVar(&cfg.StorePrefix, "store-prefix", common.StorePrefix, "the store base prefix")
 	cmd.PersistentFlags().StringVar(&cfg.StoreCertFile, "store-cert-file", "", "certificate file for client identification to the store")
 	cmd.PersistentFlags().StringVar(&cfg.StoreKeyFile, "store-key", "", "private key file for client identification to the store")
@@ -143,6 +146,7 @@ func NewKVStore(cfg *CommonConfig) (store.KVStore, error) {
 	return store.NewKVStore(store.Config{
 		Backend:       store.Backend(cfg.StoreBackend),
 		Endpoints:     cfg.StoreEndpoints,
+		Timeout:       cfg.StoreTimeout,
 		CertFile:      cfg.StoreCertFile,
 		KeyFile:       cfg.StoreKeyFile,
 		CAFile:        cfg.StoreCAFile,
@@ -195,7 +199,7 @@ func NewElection(cfg *CommonConfig, uid string) (store.Election, error) {
 		if err != nil {
 			return nil, fmt.Errorf("cannot create kv store: %v", err)
 		}
-		election = store.NewKVBackedElection(kvstore, filepath.Join(storePath, common.SentinelLeaderKey), uid)
+		election = store.NewKVBackedElection(kvstore, filepath.Join(storePath, common.SentinelLeaderKey), uid, cfg.StoreTimeout)
 	case "kubernetes":
 		kubecli, podName, namespace, err := getKubeValues(cfg)
 		if err != nil {
@@ -216,7 +220,7 @@ func getKubeValues(cfg *CommonConfig) (*kubernetes.Clientset, string, string, er
 	if err != nil {
 		return nil, "", "", err
 	}
-	kubecfg.Timeout = cluster.DefaultStoreTimeout
+	kubecfg.Timeout = cfg.StoreTimeout
 	kubecli, err := kubernetes.NewForConfig(kubecfg)
 	if err != nil {
 		return nil, "", "", fmt.Errorf("cannot create kubernetes client: %v", err)

doc/commands/stolon-keeper.md

@@ -40,7 +40,8 @@ stolon-keeper [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
       --uid string                      keeper uid (must be unique in the cluster and can contain only lower-case letters, numbers and the underscore character). If not provided a random uid will be generated.
 ```
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolon-proxy.md

@@ -29,9 +29,10 @@ stolon-proxy [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
       --tcp-keepalive-count int         set tcp keepalive probe count number
       --tcp-keepalive-idle int          set tcp keepalive idle (seconds)
       --tcp-keepalive-interval int      set tcp keepalive interval (seconds)
 ```
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolon-sentinel.md

@@ -27,6 +27,7 @@ stolon-sentinel [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl.md

@@ -28,6 +28,7 @@ stolonctl [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
@@ -43,4 +44,4 @@ stolonctl [flags]
 * [stolonctl update](stolonctl_update.md)	 - Update a cluster specification
 * [stolonctl version](stolonctl_version.md)	 - Display the version
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl_clusterdata.md

@@ -29,6 +29,7 @@ Manage current cluster data
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
@@ -37,4 +38,4 @@ Manage current cluster data
 * [stolonctl clusterdata read](stolonctl_clusterdata_read.md)	 - Retrieve the current cluster data
 * [stolonctl clusterdata write](stolonctl_clusterdata_write.md)	 - Write cluster data
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl_clusterdata_read.md

@@ -34,10 +34,11 @@ stolonctl clusterdata read [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
 
 * [stolonctl clusterdata](stolonctl_clusterdata.md)	 - Manage current cluster data
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl_clusterdata_write.md

@@ -35,10 +35,11 @@ stolonctl clusterdata write [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
 
 * [stolonctl clusterdata](stolonctl_clusterdata.md)	 - Manage current cluster data
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl_failkeeper.md

@@ -33,10 +33,11 @@ stolonctl failkeeper [keeper uid] [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
 
 * [stolonctl](stolonctl.md)	 - stolon command line client
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl_init.md

@@ -35,10 +35,11 @@ stolonctl init [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
 
 * [stolonctl](stolonctl.md)	 - stolon command line client
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl_promote.md

@@ -34,10 +34,11 @@ stolonctl promote [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
 
 * [stolonctl](stolonctl.md)	 - stolon command line client
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl_register.md

@@ -44,10 +44,11 @@ stolonctl register [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
 
 * [stolonctl](stolonctl.md)	 - stolon command line client
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl_removekeeper.md

@@ -33,10 +33,11 @@ stolonctl removekeeper [keeper uid] [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
 
 * [stolonctl](stolonctl.md)	 - stolon command line client
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl_spec.md

@@ -34,10 +34,11 @@ stolonctl spec [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
 
 * [stolonctl](stolonctl.md)	 - stolon command line client
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl_status.md

@@ -34,10 +34,11 @@ stolonctl status [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
 
 * [stolonctl](stolonctl.md)	 - stolon command line client
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl_update.md

@@ -35,10 +35,11 @@ stolonctl update [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
 
 * [stolonctl](stolonctl.md)	 - stolon command line client
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

doc/commands/stolonctl_version.md

@@ -33,10 +33,11 @@ stolonctl version [flags]
       --store-key string                private key file for client identification to the store
       --store-prefix string             the store base prefix (default "stolon/cluster")
       --store-skip-tls-verify           skip store certificate verification (insecure!!!)
+      --store-timeout duration          store request timeout (default 5s)
 ```
 
 ### SEE ALSO
 
 * [stolonctl](stolonctl.md)	 - stolon command line client
 
-###### Auto generated by spf13/cobra on 28-Jan-2020
+###### Auto generated by spf13/cobra on 6-Mar-2020

internal/store/kvbacked.go

@@ -65,6 +65,7 @@ var URLSchemeRegexp = regexp.MustCompile(`^([a-zA-Z][a-zA-Z0-9+-.]*)://`)
 type Config struct {
 	Backend       Backend
 	Endpoints     string
+	Timeout       time.Duration
 	BasePath      string
 	CertFile      string
 	KeyFile       string
@@ -172,7 +173,7 @@ func NewKVStore(cfg Config) (KVStore, error) {
 	case CONSUL, ETCDV2:
 		config := &libkvstore.Config{
 			TLS:               tlsConfig,
-			ConnectionTimeout: cluster.DefaultStoreTimeout,
+			ConnectionTimeout: cfg.Timeout,
 		}
 
 		store, err := libkv.NewStore(kvBackend, addrs, config)
@@ -190,7 +191,7 @@ func NewKVStore(cfg Config) (KVStore, error) {
 		if err != nil {
 			return nil, err
 		}
-		return &etcdV3Store{c: c, requestTimeout: cluster.DefaultStoreTimeout}, nil
+		return &etcdV3Store{c: c, requestTimeout: cfg.Timeout}, nil
 	default:
 		return nil, fmt.Errorf("Unknown store backend: %q", cfg.Backend)
 	}
@@ -344,7 +345,7 @@ func (s *KVBackedStore) GetProxiesInfo(ctx context.Context) (cluster.ProxiesInfo
 	return psi, nil
 }
 
-func NewKVBackedElection(kvStore KVStore, path, candidateUID string) Election {
+func NewKVBackedElection(kvStore KVStore, path, candidateUID string, timeout time.Duration) Election {
 	switch kvStore := kvStore.(type) {
 	case *libKVStore:
 		s := kvStore
@@ -357,7 +358,7 @@ func NewKVBackedElection(kvStore KVStore, path, candidateUID string) Election {
 			path:           path,
 			candidateUID:   candidateUID,
 			ttl:            MinTTL,
-			requestTimeout: cluster.DefaultStoreTimeout,
+			requestTimeout: timeout,
 		}
 	default:
 		panic("unknown kvstore")

tests/integration/utils.go

@@ -960,6 +960,7 @@ func NewTestEtcd(t *testing.T, dir string, backend store.Backend, a ...string) (
 	storeConfig := store.Config{
 		Backend:   store.Backend(backend),
 		Endpoints: storeEndpoints,
+		Timeout:   defaultStoreTimeout,
 	}
 	kvstore, err := store.NewKVStore(storeConfig)
 	if err != nil {
@@ -1043,6 +1044,7 @@ func NewTestConsul(t *testing.T, dir string, a ...string) (*TestStore, error) {
 	storeConfig := store.Config{
 		Backend:   store.CONSUL,
 		Endpoints: storeEndpoints,
+		Timeout:   defaultStoreTimeout,
 	}
 	kvstore, err := store.NewKVStore(storeConfig)
 	if err != nil {