Add CircleCI context retries (#467)

Author: eversC

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/aws/aws-lambda-go v1.28.0
 	github.com/aws/aws-sdk-go v1.40.34
 	github.com/beamly/go-gocd v0.0.0-20190719193049-383d56afbf92
+	github.com/cenkalti/backoff/v4 v4.1.2
 	github.com/gogo/protobuf v1.3.1 // indirect
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d // indirect

go.sum

@@ -98,6 +98,10 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
 github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/briandowns/spinner v0.0.0-20181018151057-dd69c579ff20/go.mod h1:hw/JEQBIE+c/BLI4aKM8UU8v+ZqrD3h7HC27kKt8JQU=
+github.com/cenkalti/backoff v1.1.0 h1:QnvVp8ikKCDWOsFheytRCoYWYPO/ObCTBGxT19Hc+yE=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
+github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo=
+github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=

pkg/location/circlecicontext.go

@@ -3,11 +3,14 @@ package location
 import (
 	"encoding/base64"
 	"net/http"
+	"time"
 
 	"github.com/ovotech/cloud-key-rotator/pkg/cred"
 
 	"github.com/CircleCI-Public/circleci-cli/api"
 	"github.com/CircleCI-Public/circleci-cli/settings"
+
+	"github.com/cenkalti/backoff/v4"
 )
 
 // CircleCIContext type
@@ -78,8 +81,35 @@ func (circleContext CircleCIContext) Write(serviceAccountName string, keyWrapper
 func updateCircleCIContext(contextID, envVarName, envVarValue string,
 	restClient *api.ContextRestClient) (err error) {
 
-	if err = restClient.DeleteEnvironmentVariable(contextID, envVarName); err != nil {
+	maxElapsedTimeSecs := 500
+	backoffMultiplier := 5
+
+	deleteOp := func() error {
+		logger.Infof("Deleting env var %s on contextID: %s", envVarName, contextID)
+		return restClient.DeleteEnvironmentVariable(contextID, envVarName)
+	}
+	err = callCircleCIWithExpBackoff(deleteOp,
+		time.Duration(maxElapsedTimeSecs)*time.Second,
+		float64(backoffMultiplier))
+	if err != nil {
 		return
 	}
-	return restClient.CreateEnvironmentVariable(contextID, envVarName, envVarValue)
+	createOp := func() error {
+		logger.Infof("Creating env var %s on contextID: %s", envVarName, contextID)
+		return restClient.CreateEnvironmentVariable(contextID, envVarName, envVarValue)
+	}
+	err = callCircleCIWithExpBackoff(createOp,
+		time.Duration(maxElapsedTimeSecs)*time.Second,
+		float64(backoffMultiplier))
+	return
+}
+
+// callCircleCIWithExpBackoff calls the CircleCI API with exponential backoff
+func callCircleCIWithExpBackoff(operation backoff.Operation,
+	maxElapsedTime time.Duration, multiplier float64) (err error) {
+	b := backoff.NewExponentialBackOff()
+	b.MaxElapsedTime = maxElapsedTime
+	b.Multiplier = multiplier
+	err = backoff.Retry(operation, b)
+	return
 }