Skip to content

Commit cc7e993

Browse files
mr-cmr-c
authored
Merge branch 'master' into singularity-iwdr
2 parents 087594e + d5326de commit cc7e993

17 files changed

+358
-166
lines changed

‎MANIFEST.in

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,6 @@ include tests/wf/*
77
include tests/override/*
88
include tests/checker_wf/*
99
include cwltool/schemas/v1.0/*.yml
10-
include cwltool/schemas/draft-2/*.yml
11-
include cwltool/schemas/draft-3/*.yml
12-
include cwltool/schemas/draft-3/*.md
13-
include cwltool/schemas/draft-3/salad/schema_salad/metaschema/*.yml
14-
include cwltool/schemas/draft-3/salad/schema_salad/metaschema/*.md
1510
include cwltool/schemas/v1.0/*.yml
1611
include cwltool/schemas/v1.0/*.md
1712
include cwltool/schemas/v1.0/salad/schema_salad/metaschema/*.yml
@@ -26,5 +21,20 @@ include cwltool/cwlNodeEngineWithContext.js
2621
include cwltool/extensions.yml
2722
include cwltool/jshint/jshint_wrapper.js
2823
include cwltool/jshint/jshint.js
24+
prune cwltool/schemas/v1.0/salad/typeshed
25+
prune cwltool/schemas/v1.0/salad/schema_salad/tests
26+
prune cwltool/schemas/v1.1.0-dev1/salad/typeshed
27+
prune cwltool/schemas/v1.1.0-dev1/salad/schema_salad/tests
28+
prune cwltool/schemas/presentations
29+
prune cwltool/schemas/draft-2
30+
prune cwltool/schemas/draft-1
31+
prune cwltool/schemas/draft-3
32+
prune cwltool/schemas/site
33+
prune cwltool/schemas/v1.0/examples
34+
prune cwltool/schemas/v1.0/v1.0
35+
prune cwltool/schemas/v1.1.0-dev1/examples
36+
prune cwltool/schemas/v1.1.0-dev1/v1.1.0-dev1
37+
recursive-exclude cwltool/schemas *.py
38+
2939
global-exclude *~
3040
global-exclude *.pyc

‎appveyor.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,14 @@ install:
3737

3838
build_script:
3939
- "%PYTHON%\\python.exe -m pip install -rrequirements.txt"
40+
- "%PYTHON%\\python.exe -m pip install -e ."
4041

4142
test_script:
4243
- |
43-
%PYTHON%\\python.exe setup.py test --addopts "--cov-report xml --cov cwltool -p no:cacheprovider --junit-xml=tests.xml"
44+
%PYTHON%\\python.exe -m coverage run --parallel-mode -m pytest --strict -p no:cacheprovider --junit-xml=tests.xml
45+
- "%PYTHON%\\python.exe -m coverage combine"
46+
- "%PYTHON%\\python.exe -m coverage report"
47+
- "%PYTHON%\\python.exe -m coverage xml"
4448
- "%PYTHON%\\python.exe -m codecov --file coverage.xml"
4549

4650
on_finish:

‎cwltool/main.py

Lines changed: 61 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -40,11 +40,10 @@
4040
from .loghandler import _logger, defaultStreamHandler
4141
from .mutation import MutationManager
4242
from .pack import pack
43-
from .pathmapper import (adjustDirObjs, normalizeFilesDirs, trim_listing,
44-
visit_class)
43+
from .pathmapper import adjustDirObjs, normalizeFilesDirs, trim_listing
4544
from .process import (Process, add_sizes, # pylint: disable=unused-import
4645
scandeps, shortname, use_custom_schema,
47-
use_standard_schema)
46+
use_standard_schema, CWL_IANA)
4847
from .provenance import ResearchObject
4948
from .resolver import ga4gh_tool_registries, tool_resolver
5049
from .secrets import SecretStore
@@ -53,7 +52,7 @@
5352
from .stdfsaccess import StdFsAccess
5453
from .update import ALLUPDATES, UPDATES
5554
from .utils import (DEFAULT_TMP_PREFIX, json_dumps, onWindows,
56-
processes_to_kill, versionstring,
55+
processes_to_kill, versionstring, visit_class,
5756
windows_default_container_id)
5857

5958

@@ -343,8 +342,13 @@ def init_job_order(job_order_object, # type: Optional[MutableMapping[Text
343342
job_order_object = {}
344343

345344
if print_input_deps:
346-
printdeps(job_order_object, loader, stdout, relative_deps, "",
347-
basedir=file_uri(str(input_basedir) + "/"))
345+
basedir = None
346+
uri = job_order_object["id"]
347+
if uri == args.workflow:
348+
basedir = os.path.dirname(uri)
349+
uri = ""
350+
printdeps(job_order_object, loader, stdout, relative_deps, uri,
351+
basedir=basedir, nestdirs=False)
348352
exit(0)
349353

350354
def path_to_loc(p):
@@ -388,40 +392,64 @@ def make_relative(base, obj):
388392
uri = uri_file_path(uri)
389393
obj["location"] = os.path.relpath(uri, base)
390394

391-
392-
def printdeps(obj, # type: Optional[Mapping[Text, Any]]
395+
def printdeps(obj, # type: Mapping[Text, Any]
393396
document_loader, # type: Loader
394397
stdout, # type: Union[TextIO, StreamWriter]
395398
relative_deps, # type: bool
396399
uri, # type: Text
397-
prov_args=None, # type: Any
398-
basedir=None # type: Text
399-
): # type: (...) -> Tuple[Optional[Dict[Text, Any]], Optional[Dict[Text, Any]]]
400+
basedir=None, # type: Text
401+
nestdirs=True # type: bool
402+
): # type: (...) -> None
400403
"""Print a JSON representation of the dependencies of the CWL document."""
401-
deps = {"class": "File", "location": uri} # type: Dict[Text, Any]
404+
deps = find_deps(obj, document_loader, uri, basedir=basedir,
405+
nestdirs=nestdirs)
406+
if relative_deps == "primary":
407+
base = basedir if basedir else os.path.dirname(uri_file_path(str(uri)))
408+
elif relative_deps == "cwd":
409+
base = os.getcwd()
410+
visit_class(deps, ("File", "Directory"), functools.partial(
411+
make_relative, base))
412+
stdout.write(json_dumps(deps, indent=4))
413+
414+
def prov_deps(obj, # type: Mapping[Text, Any]
415+
document_loader, # type: Loader
416+
uri, # type: Text
417+
basedir=None # type: Text
418+
): # type: (...) -> MutableMapping[Text, Any]
419+
deps = find_deps(obj, document_loader, uri, basedir=basedir)
420+
421+
def remove_non_cwl(deps): # type: (MutableMapping[Text, Any]) -> None
422+
if 'secondaryFiles' in deps:
423+
sec_files = deps['secondaryFiles']
424+
for index, entry in enumerate(sec_files):
425+
if not ('format' in entry and entry['format'] == CWL_IANA):
426+
del sec_files[index]
427+
else:
428+
remove_non_cwl(entry)
429+
430+
remove_non_cwl(deps)
431+
return deps
432+
433+
434+
def find_deps(obj, # type: Mapping[Text, Any]
435+
document_loader, # type: Loader
436+
uri, # type: Text
437+
basedir=None, # type: Text
438+
nestdirs=True # type: bool
439+
): # type: (...) -> Dict[Text, Any]
440+
"""Find the dependencies of the CWL document."""
441+
deps = {"class": "File", "location": uri, "format": CWL_IANA} # type: Dict[Text, Any]
402442

403443
def loadref(base, uri):
404444
return document_loader.fetch(document_loader.fetcher.urljoin(base, uri))
405445

406446
sfs = scandeps(
407447
basedir if basedir else uri, obj, {"$import", "run"},
408-
{"$include", "$schemas", "location"}, loadref)
448+
{"$include", "$schemas", "location"}, loadref, nestdirs=nestdirs)
409449
if sfs:
410450
deps["secondaryFiles"] = sfs
411451

412-
if relative_deps:
413-
if relative_deps == "primary":
414-
base = basedir if basedir else os.path.dirname(uri_file_path(str(uri)))
415-
elif relative_deps == "cwd":
416-
base = os.getcwd()
417-
else:
418-
raise Exception(u"Unknown relative_deps %s" % relative_deps)
419-
absdeps = copy.deepcopy(deps)
420-
visit_class(deps, ("File", "Directory"), functools.partial(make_relative, base))
421-
if prov_args:
422-
return (deps, absdeps)
423-
stdout.write(json_dumps(deps, indent=4))
424-
return (None, None)
452+
return deps
425453

426454
def print_pack(document_loader, # type: Loader
427455
processobj, # type: Union[Dict[Text, Any], List[Dict[Text, Any]]]
@@ -501,7 +529,7 @@ def main(argsl=None, # type: List[str]
501529
if not hasattr(args, key):
502530
setattr(args, key, val)
503531

504-
## Configure logging
532+
# Configure logging
505533
rdflib_logger = logging.getLogger("rdflib.term")
506534
rdflib_logger.addHandler(stderr_handler)
507535
rdflib_logger.setLevel(logging.ERROR)
@@ -562,6 +590,7 @@ def main(argsl=None, # type: List[str]
562590
runtimeContext.research_obj = ro
563591
log_file_io = ro.open_log_file_for_activity(ro.engine_uuid)
564592
prov_log_handler = logging.StreamHandler(log_file_io)
593+
565594
class ProvLogFormatter(logging.Formatter):
566595
"""Enforce ISO8601 with both T and Z."""
567596
def __init__(self): # type: () -> None
@@ -613,7 +642,8 @@ def formatTime(self, record, datefmt=None):
613642
fetcher_constructor=loadingContext.fetcher_constructor)
614643

615644
if args.print_deps:
616-
printdeps(workflowobj, document_loader, stdout, args.relative_deps, uri)
645+
printdeps(workflowobj, document_loader, stdout,
646+
args.relative_deps, uri)
617647
return 0
618648

619649
document_loader, avsc_names, processobj, metadata, uri \
@@ -752,6 +782,7 @@ def my_represent_none(self, data): # pylint: disable=unused-argument
752782
if out is not None:
753783
if runtimeContext.research_obj:
754784
runtimeContext.research_obj.create_job(out, None, True)
785+
755786
def loc_to_path(obj):
756787
for field in ("path", "nameext", "nameroot", "dirname"):
757788
if field in obj:
@@ -801,14 +832,9 @@ def loc_to_path(obj):
801832
finally:
802833
if args and runtimeContext and runtimeContext.research_obj \
803834
and workflowobj:
804-
#adding all related cwl files to RO
805835
research_obj = runtimeContext.research_obj
806-
prov_dependencies = printdeps(
807-
workflowobj, document_loader, stdout, args.relative_deps, uri,
808-
research_obj)
809-
prov_dep = prov_dependencies[1]
810-
assert prov_dep
811-
research_obj.generate_snapshot(prov_dep)
836+
prov_dependencies = prov_deps(workflowobj, document_loader, uri)
837+
research_obj.generate_snapshot(prov_dependencies)
812838
if prov_log_handler:
813839
# Stop logging so we won't half-log adding ourself to RO
814840
_logger.debug(u"[provenance] Closing provenance log file %s",

‎cwltool/pack.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,9 @@ def rewrite_id(r, mainuri):
174174
namespaces = metadata.get('$namespaces', None)
175175

176176
schemas = set() # type: Set[Text]
177+
if '$schemas' in metadata:
178+
for each_schema in metadata["$schemas"]:
179+
schemas.add(each_schema)
177180
for r in sorted(runs):
178181
dcr, metadata = document_loader.resolve_ref(r)
179182
if isinstance(dcr, CommentedSeq):
@@ -205,10 +208,15 @@ def rewrite_id(r, mainuri):
205208
import_embed(packed, set())
206209

207210
if len(packed["$graph"]) == 1:
208-
# duplicate 'cwlVersion' inside $graph when there is a single item
209-
# because we're printing contents inside '$graph' rather than whole dict
211+
# duplicate 'cwlVersion' and $schemas inside $graph when there is only
212+
# a single item because we will print the contents inside '$graph'
213+
# rather than whole dict
210214
packed["$graph"][0]["cwlVersion"] = packed["cwlVersion"]
215+
if schemas:
216+
packed["$graph"][0]["$schemas"] = list(schemas)
217+
# always include $namespaces in the #main
211218
if namespaces:
212219
packed["$graph"][0]["$namespaces"] = dict(cast(Dict, namespaces))
213220

221+
214222
return packed

‎cwltool/process.py

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -358,7 +358,7 @@ def relink(relinked, # type: Dict[Text, Text]
358358
os.unlink(path)
359359
os.symlink(os.path.relpath(link_name, path), path)
360360
else:
361-
if any(os.path.commonprefix([path, real_path]) == path \
361+
if any(os.path.commonprefix([path, real_path]) == path
362362
for path in source_directories):
363363
os.unlink(path)
364364
os.rename(real_path, path)
@@ -900,8 +900,16 @@ def mergedirs(listing):
900900
return r
901901

902902

903-
def scandeps(base, doc, reffields, urlfields, loadref, urljoin=urllib.parse.urljoin):
904-
# type: (Text, Any, Set[Text], Set[Text], Callable[[Text, Text], Any], Callable[[Text, Text], Text]) -> List[Dict[Text, Text]]
903+
CWL_IANA = "https://www.iana.org/assignments/media-types/application/cwl"
904+
905+
def scandeps(base, # type: Text
906+
doc, # type: Any
907+
reffields, # type: Set[Text]
908+
urlfields, # type: Set[Text]
909+
loadref, # type: Callable[[Text, Text], Text]
910+
urljoin=urllib.parse.urljoin, # type: Callable[[Text, Text], Text]
911+
nestdirs=True # type: bool
912+
): # type: (...) -> List[Dict[Text, Text]]
905913
r = [] # type: List[Dict[Text, Text]]
906914
if isinstance(doc, MutableMapping):
907915
if "id" in doc:
@@ -910,7 +918,8 @@ def scandeps(base, doc, reffields, urlfields, loadref, urljoin=urllib.parse.urlj
910918
if base != df:
911919
r.append({
912920
"class": "File",
913-
"location": df
921+
"location": df,
922+
"format": CWL_IANA
914923
})
915924
base = df
916925

@@ -926,44 +935,60 @@ def scandeps(base, doc, reffields, urlfields, loadref, urljoin=urllib.parse.urlj
926935
deps["listing"] = doc["listing"]
927936
if doc["class"] == "File" and "secondaryFiles" in doc:
928937
deps["secondaryFiles"] = doc["secondaryFiles"]
929-
deps = nestdir(base, deps)
938+
if nestdirs:
939+
deps = nestdir(base, deps)
930940
r.append(deps)
931941
else:
932942
if doc["class"] == "Directory" and "listing" in doc:
933-
r.extend(scandeps(base, doc["listing"], reffields, urlfields, loadref, urljoin=urljoin))
943+
r.extend(scandeps(
944+
base, doc["listing"], reffields, urlfields, loadref,
945+
urljoin=urljoin, nestdirs=nestdirs))
934946
elif doc["class"] == "File" and "secondaryFiles" in doc:
935-
r.extend(scandeps(base, doc["secondaryFiles"], reffields, urlfields, loadref, urljoin=urljoin))
947+
r.extend(scandeps(
948+
base, doc["secondaryFiles"], reffields, urlfields,
949+
loadref, urljoin=urljoin, nestdirs=nestdirs))
936950

937951
for k, v in iteritems(doc):
938952
if k in reffields:
939953
for u in aslist(v):
940954
if isinstance(u, MutableMapping):
941-
r.extend(scandeps(base, u, reffields, urlfields, loadref, urljoin=urljoin))
955+
r.extend(scandeps(
956+
base, u, reffields, urlfields, loadref,
957+
urljoin=urljoin, nestdirs=nestdirs))
942958
else:
943959
sub = loadref(base, u)
944960
subid = urljoin(base, u)
945961
deps = {
946962
"class": "File",
947-
"location": subid
963+
"location": subid,
964+
"format": CWL_IANA
948965
}
949-
sf = scandeps(subid, sub, reffields, urlfields, loadref, urljoin=urljoin)
966+
sf = scandeps(
967+
subid, sub, reffields, urlfields, loadref,
968+
urljoin=urljoin, nestdirs=nestdirs)
950969
if sf:
951970
deps["secondaryFiles"] = sf
952-
deps = nestdir(base, deps)
971+
if nestdirs:
972+
deps = nestdir(base, deps)
953973
r.append(deps)
954974
elif k in urlfields and k != "location":
955975
for u in aslist(v):
956976
deps = {
957977
"class": "File",
958978
"location": urljoin(base, u)
959979
}
960-
deps = nestdir(base, deps)
980+
if nestdirs:
981+
deps = nestdir(base, deps)
961982
r.append(deps)
962983
elif k not in ("listing", "secondaryFiles"):
963-
r.extend(scandeps(base, v, reffields, urlfields, loadref, urljoin=urljoin))
984+
r.extend(scandeps(
985+
base, v, reffields, urlfields, loadref, urljoin=urljoin,
986+
nestdirs=nestdirs))
964987
elif isinstance(doc, MutableSequence):
965988
for d in doc:
966-
r.extend(scandeps(base, d, reffields, urlfields, loadref, urljoin=urljoin))
989+
r.extend(scandeps(
990+
base, d, reffields, urlfields, loadref, urljoin=urljoin,
991+
nestdirs=nestdirs))
967992

968993
if r:
969994
normalizeFilesDirs(r)

0 commit comments

Comments
 (0)