Merge pull request #1486 from BD2KGenomics/issues/1485-cluster-stats-fix

Fix typos in cluster stats mechanism
DataBiosphere · Feb 7, 2017 · 60bfa1c · 60bfa1c
2 parents 1ea068f + 12b31a0
commit 60bfa1c
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 6 deletions.
diff --git a/src/toil/batchSystems/abstractBatchSystem.py b/src/toil/batchSystems/abstractBatchSystem.py
@@ -320,23 +320,31 @@ def workerCleanup(info):
 
 class NodeInfo(object):
     """
-    The cores attribute  is a floating point value between 0 (all cores idle) and 1 (all cores
+    The coresUsed attribute  is a floating point value between 0 (all cores idle) and 1 (all cores
     busy), reflecting the CPU load of the node.
 
-    The memory attribute is a floating point value between 0 (no memory used) and 1 (all memory
+    The memoryUsed attribute is a floating point value between 0 (no memory used) and 1 (all memory
     used), reflecting the memory pressure on the node.
 
+    The coresTotal and memoryTotal attributes are the node's resources, not just the used resources
+
+    The requestedCores and requestedMemory attributes are all the resources that Toil Jobs have reserved on the
+    node, regardless of whether the resources are actually being used by the Jobs.
+
     The workers attribute is an integer reflecting the number of workers currently active workers
     on the node.
     """
     def __init__(self, coresUsed, memoryUsed, coresTotal, memoryTotal,
                  requestedCores, requestedMemory, workers):
         self.coresUsed = coresUsed
-        self.totalCores = coresTotal
         self.memoryUsed = memoryUsed
+
+        self.coresTotal = coresTotal
         self.memoryTotal = memoryTotal
+
         self.requestedCores = requestedCores
         self.requestedMemory = requestedMemory
+
         self.workers = workers
 
 

diff --git a/src/toil/provisioners/abstractProvisioner.py b/src/toil/provisioners/abstractProvisioner.py
@@ -100,8 +100,8 @@ def checkStats(self):
     def _gatherStats(self, preemptable):
         def toDict(nodeInfo):
             # convert NodeInfo object to dict to improve JSON output
-            return dict(memory=nodeInfo.memoryUsage,
-                        cores=nodeInfo.coresUsage,
+            return dict(memory=nodeInfo.memoryUsed,
+                        cores=nodeInfo.coresUsed,
                         memoryTotal=nodeInfo.memoryTotal,
                         coresTotal=nodeInfo.coresTotal,
                         requestedCores=nodeInfo.requestedCores,

diff --git a/src/toil/test/provisioners/aws/awsProvisionerTest.py b/src/toil/test/provisioners/aws/awsProvisionerTest.py
@@ -175,7 +175,9 @@ def _getScript(self):
         os.unlink(sseKeyFile)
 
     def _runScript(self, toilOptions):
-        runCommand = ['/home/venv/bin/python', '/home/sort.py', '--fileToSort=/home/keyFile']
+        # the file to sort is included in the Toil appliance so we know it will be on every node in the cluster
+        # hacky, but it works.
+        runCommand = ['/home/venv/bin/python', '/home/sort.py', '--fileToSort=/home/s3am/bin/asadmin']
         runCommand.extend(toilOptions)
         runCommand.append('--sseKey=/home/keyFile')
         self.sshUtil(runCommand)