Merge pull request #898 from wtsi-hgi/rm-intermediates

Delete intermediate outputs even when they exist

Author: mr-c

cwltool/executors.py

@@ -5,7 +5,7 @@
 import tempfile
 import threading
 from abc import ABCMeta, abstractmethod
-from typing import Any, Dict, List, Optional, Set, Tuple
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple
 
 import psutil
 from schema_salad.validate import ValidationException
@@ -97,7 +97,11 @@ def execute(self,
                 path_mapper=runtime_context.path_mapper)
 
         if runtime_context.rm_tmpdir:
-            cleanIntermediate(self.output_dirs)
+            if runtime_context.cachedir is None:
+                output_dirs = self.output_dirs # type: Iterable[Any]
+            else:
+                output_dirs = filter(lambda x: not x.startswith(runtime_context.cachedir), self.output_dirs)
+            cleanIntermediate(output_dirs)
 
         if self.final_output and self.final_status:
 

cwltool/process.py

@@ -372,9 +372,9 @@ def relink(relinked,  # type: Dict[Text, Text]
     return outputObj
 
 
-def cleanIntermediate(output_dirs):  # type: (Set[Text]) -> None
+def cleanIntermediate(output_dirs):  # type: (Iterable[Text]) -> None
     for a in output_dirs:
-        if os.path.exists(a) and empty_subtree(a):
+        if os.path.exists(a):
             _logger.debug(u"Removing intermediate output directory %s", a)
             shutil.rmtree(a, True)