Add more unit test cases for cluster manager

Author: amat27

hpc_cluster_manager.py

@@ -123,9 +123,7 @@ def on_slave_heartbeat(self, hostname, now=datetime.utcnow()):
             if self._heart_beat_table[u_hostname].state == HpcState.Provisioning:
                 with self._table_lock:
                     if self._heart_beat_table[u_hostname].state == HpcState.Provisioning:
-                        self._heart_beat_table[u_hostname] = self._heart_beat_table[u_hostname]._replace(
-                            state=HpcState.Configuring)
-                        self.logger.info("Configuring Host {}".format(u_hostname))
+                        self._set_nodes_configuring(u_hostname)
         else:
             self.logger.error("Host {} is not recognized. Heartbeat ignored.".format(u_hostname))
             self.logger.debug("_table {} ".format(self._heart_beat_table))
@@ -197,6 +195,7 @@ def _get_nodes_name_in_state(self, state):
         # type: (HpcState) -> [str]
         return [host.hostname for host in dict(self._heart_beat_table).itervalues() if host.state == state]
 
+    # TODO:  make state_machine methods more testable
     def _configure_compute_nodes_state_machine(self):
         # type: () -> ()
         configuring_node_names = self._get_nodes_name_in_state(HpcState.Configuring)
@@ -229,11 +228,11 @@ def _configure_compute_nodes_state_machine(self):
         invalid_state_node_dict = {}
         for node_status in node_status_list:
             node_name = _get_node_name_from_status(node_status)
-            node_state = node_status[HpcRestClient.NODE_STATUS_NODE_STATE_KEY]
+            node_state = _get_node_state_from_status(node_status)
             if _check_node_health_unapproved(node_status):
                 unapproved_node_list.append(node_name)
             # node approved
-            elif (self._check_node_in_mesos_group(node_status) or (
+            elif (not self._check_node_in_mesos_group(node_status) or (
                     self._node_group_specified() and self._check_node_in_specified_group(node_status))):
                 if _check_node_state_online(node_status):
                     take_offline_node_list.append(node_name)
@@ -372,6 +371,10 @@ def _set_nodes_running(self, node_names):
         # type: (Iterable[str]) -> ()
         self._set_node_state(node_names, HpcState.Running, "Running")
 
+    def _set_nodes_configuring(self, node_names):
+        # type: (Iterable[str]) -> ()
+        self._set_node_state(node_names, HpcState.Configuring, "Configuring")
+
     def _set_node_state(self, node_names, node_state, state_name):
         # type: (Iterable[str], int, str) -> [(str, int)]
         set_nodes = []
@@ -478,7 +481,7 @@ def _close_node_state_machine(self, node_names):
 
     def _check_node_in_mesos_group(self, node_status):
         # type: (dict[str, any]) -> bool
-        return self.MESOS_NODE_GROUP_NAME.upper() not in _upper_strings(
+        return self.MESOS_NODE_GROUP_NAME.upper() in _upper_strings(
             node_status[HpcRestClient.NODE_STATUS_NODE_GROUP_KEY])
 
     def _check_node_in_specified_group(self, node_status):

hpc_cluster_manger_unittest.py

@@ -1,9 +1,10 @@
 import datetime
 import unittest
 
-from mock import patch
+from mock import patch, MagicMock
 
 from hpc_cluster_manager import HpcClusterManager, HpcState
+from restclient import HpcRestClient
 
 HOST1HOSTNAME = "host1hostname"
 HOST1FQDN = "host1hostname.fqdn.com"
@@ -17,6 +18,8 @@
 HOST2AGENTID = "host2agentid"
 HOST2TASKID1 = "host2taskid1"
 
+HOST3HOSTNAME = "host3hostname"
+
 ZERODELTA = datetime.timedelta(0)
 
 
@@ -37,6 +40,15 @@ def dst(self, dt):
 ONESEC = datetime.timedelta(seconds=1)
 
 
+def _set_node_name(node_status, name):
+    node_status[HpcRestClient.NODE_STATUS_NODE_NAME_KEY] = name
+
+
+_true_mock_result = MagicMock(return_value=True)
+_false_mock_result = MagicMock(return_value=False)
+_empty_mock_result = MagicMock(return_value="")
+
+
 class HeartbeatTableUnitTest(unittest.TestCase):
     def setUp(self):
         pass
@@ -228,6 +240,17 @@ def test_heartbeat_time_out_positive(self, mock_restc):
         _, res, _ = clusmgr._check_timeout(UTCNOW + TENMINUTES)
         self.assertTrue(res)
 
+    @patch("hpc_cluster_manager.HpcRestClient", autospec=True)
+    def test_invalid_heartbeat(self, mock_restc):
+        clusmgr = HpcClusterManager(mock_restc)
+        clusmgr.on_slave_heartbeat(HOST1HOSTNAME)
+        self.assertFalse(clusmgr._heart_beat_table)
+
+    @patch("hpc_cluster_manager.HpcRestClient", autospec=True)
+    def test_missing_task_info(self, mock_restc):
+        clusmgr = HpcClusterManager(mock_restc)
+        self.assertEqual(clusmgr.get_task_info(HOST1HOSTNAME), ("", ""))
+
 
 if __name__ == '__main__':
     unittest.main()