Merge pull request #1242 from common-workflow-language/control-cache

Control doc cache

Author: mr-c

cwltool/argparser.py

@@ -340,6 +340,22 @@ def arg_parser() -> argparse.ArgumentParser:
         dest="skip_schemas",
     )
 
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument(
+        "--no-doc-cache",
+        action="store_false",
+        help="Disable disk cache for documents loaded over HTTP",
+        default=True,
+        dest="doc_cache",
+    )
+    exgroup.add_argument(
+        "--doc-cache",
+        action="store_true",
+        help="Enable disk cache for documents loaded over HTTP",
+        default=True,
+        dest="doc_cache",
+    )
+
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--verbose", action="store_true", help="Default logging")
     exgroup.add_argument(

cwltool/context.py

@@ -71,6 +71,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None:
         self.prov_obj = None  # type: Optional[ProvenanceProfile]
         self.do_update = None  # type: Optional[bool]
         self.jobdefaults = None  # type: Optional[CommentedMap]
+        self.doc_cache = True  # type: bool
 
         super(LoadingContext, self).__init__(kwargs)
 

cwltool/load_tool.py

@@ -64,12 +64,13 @@
 ResolverType = Callable[[Loader, Union[str, Dict[str, Any]]], str]
 
 
-def default_loader(fetcher_constructor=None, enable_dev=False):
-    # type: (Optional[FetcherConstructorType], bool) -> Loader
+def default_loader(fetcher_constructor=None, enable_dev=False, doc_cache=True):
+    # type: (Optional[FetcherConstructorType], bool, bool) -> Loader
     return Loader(
         jobloaderctx,
         fetcher_constructor=fetcher_constructor,
         allow_attachments=lambda r: enable_dev,
+        doc_cache=doc_cache,
     )
 
 
@@ -113,7 +114,11 @@ def fetch_document(
     else:
         loadingContext = loadingContext.copy()
         if loadingContext.loader is None:
-            loadingContext.loader = default_loader(loadingContext.fetcher_constructor)
+            loadingContext.loader = default_loader(
+                loadingContext.fetcher_constructor,
+                enable_dev=loadingContext.enable_dev,
+                doc_cache=loadingContext.doc_cache,
+            )
 
     if isinstance(argsworkflow, str):
         uri, fileuri = resolve_tool_uri(
@@ -321,6 +326,7 @@ def resolve_and_validate_document(
         cache=sch_document_loader.cache,
         fetcher_constructor=loadingContext.fetcher_constructor,
         skip_schemas=skip_schemas,
+        doc_cache=loadingContext.doc_cache,
     )
 
     if cwlVersion == "v1.0":

cwltool/main.py

@@ -667,7 +667,9 @@ def setup_loadingContext(
     else:
         loadingContext = loadingContext.copy()
     loadingContext.loader = default_loader(
-        loadingContext.fetcher_constructor, enable_dev=args.enable_dev
+        loadingContext.fetcher_constructor,
+        enable_dev=args.enable_dev,
+        doc_cache=args.doc_cache,
     )
     loadingContext.research_obj = runtimeContext.research_obj
     loadingContext.disable_js_validation = args.disable_js_validation or (

setup.py

@@ -61,7 +61,7 @@
         "ruamel.yaml >= 0.12.4, <= 0.16.5",
         "rdflib >= 4.2.2, < 4.3.0",
         "shellescape >= 3.4.1, < 3.5",
-        "schema-salad >= 5, < 6",
+        "schema-salad >= 5.0.20200126033820, < 6",
         "mypy-extensions",
         "psutil",
         "prov == 1.5.1",

tests/test_pack.py

@@ -47,7 +47,9 @@ def test_pack():
 
 
 def test_pack_input_named_name():
-    loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/trick_revsort.cwl"))
+    loadingContext, workflowobj, uri = fetch_document(
+        get_data("tests/wf/trick_revsort.cwl")
+    )
     loadingContext.do_update = False
     loadingContext, uri = resolve_and_validate_document(
         loadingContext, workflowobj, uri