PowerFlex/ScaleIO - Wait after SDC service start/restart/stop, and retry to fetch SDC id/guid

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptor.java

@@ -37,6 +37,7 @@
 import org.apache.cloudstack.utils.qemu.QemuImgException;
 import org.apache.cloudstack.utils.qemu.QemuImgFile;
 import org.apache.cloudstack.utils.qemu.QemuObject;
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.io.filefilter.WildcardFileFilter;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.LogManager;
@@ -581,14 +582,23 @@ public Ternary<Boolean, Map<String, String>, String> prepareStorageClient(Storag
         }
 
         if (!ScaleIOUtil.isSDCServiceActive()) {
+            logger.debug("SDC service is not active on host, starting it");
             if (!ScaleIOUtil.startSDCService()) {
                 return new Ternary<>(false, null, "Couldn't start SDC service on host");
             }
-        } else if (!ScaleIOUtil.restartSDCService()) {
-            return new Ternary<>(false, null, "Couldn't restart SDC service on host");
+        } else {
+            logger.debug("SDC service is active on host, re-starting it");
+            if (!ScaleIOUtil.restartSDCService()) {
+                return new Ternary<>(false, null, "Couldn't restart SDC service on host");
+            }
+        }
+
+        Map<String, String> sdcDetails = getSDCDetails(details);
+        if (MapUtils.isEmpty(sdcDetails)) {
+            return new Ternary<>(false, null, "Couldn't get the SDC details on the host");
         }
 
-        return new Ternary<>( true, getSDCDetails(details), "Prepared client successfully");
+        return new Ternary<>( true, sdcDetails, "Prepared client successfully");
     }
 
     public Pair<Boolean, String> unprepareStorageClient(Storage.StoragePoolType type, String uuid) {
@@ -611,20 +621,37 @@ public Pair<Boolean, String> unprepareStorageClient(Storage.StoragePoolType type
 
     private Map<String, String> getSDCDetails(Map<String, String> details) {
         Map<String, String> sdcDetails = new HashMap<String, String>();
-        if (details == null || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID))  {
+        if (MapUtils.isEmpty(details)  || !details.containsKey(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID))  {
             return sdcDetails;
         }
 
         String storageSystemId = details.get(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID);
-        String sdcId = ScaleIOUtil.getSdcId(storageSystemId);
-        if (sdcId != null) {
-            sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId);
-        } else {
-            String sdcGuId = ScaleIOUtil.getSdcGuid();
-            if (sdcGuId != null) {
-                sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId);
-            }
+        if (StringUtils.isEmpty(storageSystemId)) {
+            return sdcDetails;
         }
+
+        int waitTimeInSecs = 5;
+        int timeBetweenTries = 1000; // Try more frequently (every sec) and return early when SDC Id or Guid found
+        do {
+            String sdcId = ScaleIOUtil.getSdcId(storageSystemId);
+            if (sdcId != null) {
+                sdcDetails.put(ScaleIOGatewayClient.SDC_ID, sdcId);
+                return sdcDetails;
+            } else {
+                String sdcGuId = ScaleIOUtil.getSdcGuid();
+                if (sdcGuId != null) {
+                    sdcDetails.put(ScaleIOGatewayClient.SDC_GUID, sdcGuId);
+                    return sdcDetails;
+                }
+            }
+
+            try {
+                Thread.sleep(timeBetweenTries);
+            } catch (Exception ignore) {
+            }
+            waitTimeInSecs--;
+        } while (waitTimeInSecs > 0);
+
         return sdcDetails;
     }
 

plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/ScaleIOStorageAdaptorTest.java

@@ -116,9 +116,9 @@ public void testPrepareStorageClient_SDCServiceRestarted() {
 
         Ternary<Boolean, Map<String, String>, String> result = scaleIOStorageAdaptor.prepareStorageClient(Storage.StoragePoolType.PowerFlex, poolUuid, new HashMap<>());
 
-        Assert.assertTrue(result.first());
-        Assert.assertNotNull(result.second());
-        Assert.assertTrue(result.second().isEmpty());
+        Assert.assertFalse(result.first());
+        Assert.assertNull(result.second());
+        Assert.assertEquals("Couldn't get the SDC details on the host", result.third());
     }
 
     @Test

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/provider/ScaleIOHostListener.java

@@ -102,12 +102,12 @@ private String getSdcIdOfHost(HostVO host, StoragePool storagePool) {
         if (systemId == null) {
             throw new CloudRuntimeException("Failed to get the system id for PowerFlex storage pool " + storagePool.getName());
         }
-        Map<String,String> details = new HashMap<>();
+        Map<String, String> details = new HashMap<>();
         details.put(ScaleIOGatewayClient.STORAGE_POOL_SYSTEM_ID, systemId);
 
         ModifyStoragePoolCommand cmd = new ModifyStoragePoolCommand(true, storagePool, storagePool.getPath(), details);
         ModifyStoragePoolAnswer answer  = sendModifyStoragePoolCommand(cmd, storagePool, host);
-        Map<String,String> poolDetails = answer.getPoolInfo().getDetails();
+        Map<String, String> poolDetails = answer.getPoolInfo().getDetails();
         if (MapUtils.isEmpty(poolDetails)) {
             String msg = String.format("PowerFlex storage SDC details not found on the host: %s, (re)install SDC and restart agent", host);
             logger.warn(msg);

plugins/storage/volume/scaleio/src/main/java/org/apache/cloudstack/storage/datastore/util/ScaleIOUtil.java

@@ -216,16 +216,38 @@ public static boolean enableSDCService() {
 
     public static boolean startSDCService() {
         int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_START_CMD);
-        return exitValue == 0;
+        if (exitValue != 0) {
+            return false;
+        }
+        waitForSecs(3);
+        return true;
     }
 
     public static boolean stopSDCService() {
         int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_STOP_CMD);
-        return exitValue == 0;
+        if (exitValue != 0) {
+            return false;
+        }
+        waitForSecs(1);
+        return true;
     }
 
     public static boolean restartSDCService() {
         int exitValue = Script.runSimpleBashScriptForExitValue(SDC_SERVICE_RESTART_CMD);
-        return exitValue == 0;
+        if (exitValue != 0) {
+            return false;
+        }
+        waitForSecs(3);
+        return true;
+    }
+
+    private static void waitForSecs(long waitTimeInSecs) {
+        if (waitTimeInSecs < 0) {
+            waitTimeInSecs = 1;
+        }
+        try {
+            Thread.sleep(waitTimeInSecs * 1000);
+        } catch (InterruptedException ignore) {
+        }
     }
 }