Merge pull request #1226 from andyzhangx/increase-azcopy-timeout

fix: increase volume cloning timeout and reduce time cost

Author: andyzhangx

pkg/blob/blob.go

@@ -176,6 +176,7 @@ type DriverOptions struct {
 	EnableAznfsMount                       bool
 	VolStatsCacheExpireInMinutes           int
 	SasTokenExpirationMinutes              int
+	WaitForAzCopyTimeoutMinutes            int
 	EnableVolumeMountGroup                 bool
 	FSGroupChangePolicy                    string
 }
@@ -195,6 +196,7 @@ func (option *DriverOptions) AddFlags() {
 	flag.BoolVar(&option.EnableAznfsMount, "enable-aznfs-mount", false, "replace nfs mount with aznfs mount")
 	flag.IntVar(&option.VolStatsCacheExpireInMinutes, "vol-stats-cache-expire-in-minutes", 10, "The cache expire time in minutes for volume stats cache")
 	flag.IntVar(&option.SasTokenExpirationMinutes, "sas-token-expiration-minutes", 1440, "sas token expiration minutes during volume cloning")
+	flag.IntVar(&option.WaitForAzCopyTimeoutMinutes, "wait-for-azcopy-timeout-minutes", 18, "timeout in minutes for waiting for azcopy to finish")
 	flag.BoolVar(&option.EnableVolumeMountGroup, "enable-volume-mount-group", true, "indicates whether enabling VOLUME_MOUNT_GROUP")
 	flag.StringVar(&option.FSGroupChangePolicy, "fsgroup-change-policy", "", "indicates how the volume's ownership will be changed by the driver, OnRootMismatch is the default value")
 }
@@ -239,6 +241,8 @@ type Driver struct {
 	azcopySasTokenCache azcache.Resource
 	// sas expiry time for azcopy in volume clone
 	sasTokenExpirationMinutes int
+	// timeout in minutes for waiting for azcopy to finish
+	waitForAzCopyTimeoutMinutes int
 	// azcopy for provide exec mock for ut
 	azcopy *util.Azcopy
 }
@@ -261,6 +265,7 @@ func NewDriver(options *DriverOptions, kubeClient kubernetes.Interface, cloud *p
 		mountPermissions:                       options.MountPermissions,
 		enableAznfsMount:                       options.EnableAznfsMount,
 		sasTokenExpirationMinutes:              options.SasTokenExpirationMinutes,
+		waitForAzCopyTimeoutMinutes:            options.WaitForAzCopyTimeoutMinutes,
 		fsGroupChangePolicy:                    options.FSGroupChangePolicy,
 		azcopy:                                 &util.Azcopy{},
 		KubeClient:                             kubeClient,

pkg/blob/blob_test.go

@@ -50,12 +50,13 @@ const (
 
 func NewFakeDriver() *Driver {
 	driverOptions := DriverOptions{
-		NodeID:                  fakeNodeID,
-		DriverName:              DefaultDriverName,
-		BlobfuseProxyEndpoint:   "",
-		EnableBlobfuseProxy:     false,
-		BlobfuseProxyConnTimout: 5,
-		EnableBlobMockMount:     false,
+		NodeID:                      fakeNodeID,
+		DriverName:                  DefaultDriverName,
+		BlobfuseProxyEndpoint:       "",
+		EnableBlobfuseProxy:         false,
+		BlobfuseProxyConnTimout:     5,
+		WaitForAzCopyTimeoutMinutes: 1,
+		EnableBlobMockMount:         false,
 	}
 	driver := NewDriver(&driverOptions, nil, &azure.Cloud{})
 	driver.Name = fakeDriverName
@@ -79,12 +80,13 @@ func TestNewFakeDriver(t *testing.T) {
 
 func TestNewDriver(t *testing.T) {
 	driverOptions := DriverOptions{
-		NodeID:                  fakeNodeID,
-		DriverName:              DefaultDriverName,
-		BlobfuseProxyEndpoint:   "",
-		EnableBlobfuseProxy:     false,
-		BlobfuseProxyConnTimout: 5,
-		EnableBlobMockMount:     false,
+		NodeID:                      fakeNodeID,
+		DriverName:                  DefaultDriverName,
+		BlobfuseProxyEndpoint:       "",
+		EnableBlobfuseProxy:         false,
+		BlobfuseProxyConnTimout:     5,
+		WaitForAzCopyTimeoutMinutes: 1,
+		EnableBlobMockMount:         false,
 	}
 	driver := NewDriver(&driverOptions, nil, &azure.Cloud{})
 	fakedriver := NewFakeDriver()

pkg/blob/controllerserver.go

@@ -62,8 +62,7 @@ const (
 	SPN                             = "SPN"
 	authorizationPermissionMismatch = "AuthorizationPermissionMismatch"
 
-	waitForCopyInterval = 5 * time.Second
-	waitForCopyTimeout  = 3 * time.Minute
+	waitForAzCopyInterval = 2 * time.Second
 )
 
 // CreateVolume provisions a volume
@@ -757,8 +756,8 @@ func (d *Driver) copyBlobContainer(req *csi.CreateVolumeRequest, accountSasToken
 		return fmt.Errorf("srcContainerName(%s) or dstContainerName(%s) is empty", srcContainerName, dstContainerName)
 	}
 
-	timeAfter := time.After(waitForCopyTimeout)
-	timeTick := time.Tick(waitForCopyInterval)
+	timeAfter := time.After(time.Duration(d.waitForAzCopyTimeoutMinutes) * time.Minute)
+	timeTick := time.Tick(waitForAzCopyInterval)
 	srcPath := fmt.Sprintf("https://%s.blob.%s/%s%s", accountName, storageEndpointSuffix, srcContainerName, accountSasToken)
 	dstPath := fmt.Sprintf("https://%s.blob.%s/%s%s", accountName, storageEndpointSuffix, dstContainerName, accountSasToken)