Scandeps only picks up file objects when "location" in urlfileds. (#194)

* Scandeps only picks up file objects when "location" in urlfileds.

* Make test for scandeps actually test something.  Also expand pack() test to
include external (non-packable) files.

Author: tetron

cwltool/main.py

@@ -461,6 +461,14 @@ def pathToLoc(p):
 
     return (job_order_object, input_basedir)
 
+def makeRelative(base, ob):
+    u = ob.get("location", ob.get("path"))
+    if ":" in u.split("/")[0] and not u.startswith("file://"):
+        pass
+    else:
+        if u.startswith("file://"):
+            u = u[7:]
+        ob["location"] = os.path.relpath(u, base)
 
 def printdeps(obj, document_loader, stdout, relative_deps, uri, basedir=None):
     # type: (Dict[Text, Any], Loader, IO[Any], bool, Text, Text) -> None
@@ -472,7 +480,7 @@ def loadref(b, u):
 
     sf = scandeps(
         basedir if basedir else uri, obj, set(("$import", "run")),
-        set(("$include", "$schemas")), loadref)
+        set(("$include", "$schemas", "location")), loadref)
     if sf:
         deps["secondaryFiles"] = sf
 
@@ -484,14 +492,8 @@ def loadref(b, u):
         else:
             raise Exception(u"Unknown relative_deps %s" % relative_deps)
 
-        def makeRelative(ob):
-            u = ob.get("location", ob.get("path"))
-            if ":" in u.split("/")[0] and not u.startswith("file://"):
-                pass
-            else:
-                ob["location"] = os.path.relpath(u, base)
-        adjustFileObjs(deps, makeRelative)
-        adjustDirObjs(deps, makeRelative)
+        adjustFileObjs(deps, functools.partial(makeRelative, base))
+        adjustDirObjs(deps, functools.partial(makeRelative, base))
 
     stdout.write(json.dumps(deps, indent=4))
 

cwltool/process.py

@@ -645,7 +645,7 @@ def scandeps(base, doc, reffields, urlfields, loadref):
                     })
                     base = df
 
-        if doc.get("class") in ("File", "Directory"):
+        if doc.get("class") in ("File", "Directory") and "location" in urlfields:
             u = doc.get("location", doc.get("path"))
             if u:
                 deps = {

tests/test_examples.py

@@ -122,16 +122,16 @@ def test_scandeps(self):
                         "id": "file:///example/foo.cwl#input1",
                         "default": {
                             "class": "File",
-                            "path": "file:///example/data.txt"
+                            "location": "file:///example/data.txt"
                         }
                     }],
                     "run": {
                         "id": "file:///example/bar.cwl",
                         "inputs": [{
                             "id": "file:///example/bar.cwl#input2",
                             "default": {
-                                "class": "File",
-                                "path": "file:///example/data2.txt"
+                                "class": "Directory",
+                                "location": "file:///example/data2"
                             }
                         }]
                     }
@@ -145,10 +145,41 @@ def loadref(base, p):
             else:
                 raise Exception("test case can't load things")
 
-        print json.dumps(cwltool.process.scandeps(obj["id"], obj,
+        sc = cwltool.process.scandeps(obj["id"], obj,
                                        set(("$import", "run")),
-                                       set(("$include", "$schemas", "path")),
-                                                  loadref), indent=4)
+                                       set(("$include", "$schemas", "location")),
+                                                  loadref)
+
+        sc.sort(key=lambda k: k["basename"])
+
+        self.assertEquals([{
+            "basename": "bar.cwl",
+            "class": "File",
+            "location": "file:///example/bar.cwl"
+        },
+        {
+            "basename": "data.txt",
+            "class": "File",
+            "location": "file:///example/data.txt"
+        },
+        {
+            "basename": "data2",
+            "class": "Directory",
+            "location": "file:///example/data2"
+        }], sc)
+
+        sc = cwltool.process.scandeps(obj["id"], obj,
+                                       set(("run"),),
+                                       set(), loadref)
+
+        sc.sort(key=lambda k: k["basename"])
+
+        self.assertEquals([{
+            "basename": "bar.cwl",
+            "class": "File",
+            "location": "file:///example/bar.cwl"
+        }], sc)
+
 
 class TestTypeCompare(unittest.TestCase):
     def test_typecompare(self):

tests/test_pack.py

@@ -1,8 +1,14 @@
 import unittest
 import json
+import os
+from functools import partial
+
 from cwltool.load_tool import fetch_document, validate_document
 import cwltool.pack
 import cwltool.workflow
+from cwltool.main import makeRelative
+from cwltool.process import adjustFileObjs, adjustDirObjs
+
 
 class TestPack(unittest.TestCase):
     def test_pack(self):
@@ -14,4 +20,7 @@ def test_pack(self):
         packed = cwltool.pack.pack(document_loader, processobj, uri, metadata)
         with open("tests/wf/expect_packed.cwl") as f:
             expect_packed = json.load(f)
+        adjustFileObjs(packed, partial(makeRelative, os.path.abspath("tests/wf")))
+        adjustDirObjs(packed, partial(makeRelative, os.path.abspath("tests/wf")))
+
         self.assertEqual(expect_packed, packed)

tests/wf/expect_packed.cwl

@@ -6,7 +6,11 @@
                 {
                     "doc": "The input file to be processed.",
                     "type": "File",
-                    "id": "#main/input"
+                    "id": "#main/input",
+                    "default": {
+                      "class": "File",
+                      "location": "hello.txt"
+                    }
                 },
                 {
                     "default": true,

tests/wf/hello.txt

@@ -0,0 +1,6 @@
+Hello
+world
+testing
+one
+two
+three.

tests/wf/revsort.cwl

@@ -23,6 +23,9 @@ inputs:
   input:
     type: File
     doc: "The input file to be processed."
+    default:
+      class: File
+      location: hello.txt
   reverse_sort:
     type: boolean
     default: true