Implement a better config file system for CWL/WDL options

contrib/mypy-stubs/configargparse/__init__.pyi

@@ -1,3 +1,5 @@
 from .configargparse import ArgParser as ArgParser
 from .configargparse import ArgumentParser as ArgumentParser
+from .configargparse import Namespace as Namespace
+from .configargparse import SUPPRESS as SUPPRESS
 from .configargparse import YAMLConfigFileParser as YAMLConfigFileParser

contrib/mypy-stubs/configargparse/configargparse.pyi

@@ -1,5 +1,5 @@
 import argparse
-from typing import Any, OrderedDict, Sequence, TypeVar
+from typing import Any, List, OrderedDict, Sequence, TypeVar
 
 __all__ = ["ArgumentParser", "YAMLConfigFileParser", "ConfigFileParser"]
 _N = TypeVar("_N")
@@ -19,10 +19,16 @@ class YAMLConfigFileParser(ConfigFileParser):
 class ArgumentParser(argparse.ArgumentParser):
     @property
     def _config_file_parser(self) -> Any: ...
+    @property
+    def _default_config_files(self) -> List[Any]: ...
+    @property
+    def _ignore_unknown_config_file_keys(self) -> Any: ...
+
     def __init__(self, *args: Any, **kwargs: Any) -> None: ...
     # There may be a better way of type hinting this without a type: ignore, but mypy gets unhappy pretty much no matter what as the signatures for parse_args doesn't match with its superclass in argparse
     def parse_args(self, args: Sequence[str] | None = None, namespace: Namespace | None = None, config_file_contents: str | None = None, env_vars: Any = None) -> Namespace: ...  # type: ignore[override]
     def get_source_to_settings_dict(self) -> OrderedDict[Any, Any]: ...
 
 Namespace = argparse.Namespace
 ArgParser = ArgumentParser
+SUPPRESS = argparse.SUPPRESS

docs/running/cliOptions.rst

@@ -16,19 +16,45 @@ The Config File
 Instead of changing the arguments on the CLI, Toil offers support for using a configuration file (Note: Support for the
 configuration file and environmental variables require the use of ``configargparse``).
 
-To generate a default configuration file::
+Options will be applied with priority:
 
-    $ toil config [file].yaml
+  1. CLI
+
+  2. Environmental Variables
+
+  3. Config file values
+
+     a. Provided config file through CLI
+
+     b. Default config value in ``$HOME/.toil/default.yaml``
+
+  4. Defaults
+
+To generate a configuration file, run::
+
+    $ toil config [filename].yaml
+
+Then uncomment options as necessary and change/provide new values. 
 
 After editing the config file, make Toil take in the new options::
 
-    $ python example.py --config=[file].yaml
+    $ python example.py --config=[filename].yaml
+
+Alternatively, you can edit the default config file, which is located at ``$HOME/.toil/default.yaml``
+
+If CLI options are used in addition to the configuration file, the CLI options will overwrite the configuration file
+options. For example::
+
+    $ python example.py --config=[filename].yaml --maxNodes 20
+
+With configuration file::
+
+    ...
+    maxNodes=[10]
+    ...
 
-If CLI options are used in addition with the configuration file, the CLI options will overwrite the configuration file
-options::
+Will result in maxNodes=[20].
 
-    $ python example.py --config=[file].yaml --maxNodes 20
-    # maxNodes=[20] even though default maxNodes=[10]
 
 The Job Store
 -------------

requirements.txt

@@ -11,5 +11,6 @@ pytz>=2012
 enlighten>=1.5.2, <2
 configargparse>=1.7,<2
 ruamel.yaml>=0.15
+ruamel.yaml.string>=0.1.1
 pyyaml>=6,<7
 typing-extensions>=4.6.2, <5

src/toil/batchSystems/kubernetes.py

@@ -104,7 +104,8 @@
 from toil.batchSystems.cleanup_support import BatchSystemCleanupSupport
 from toil.batchSystems.contained_executor import pack_job
 from toil.batchSystems.options import OptionSetter
-from toil.common import SYS_MAX_SIZE, Config, Toil
+from toil.common import Config, Toil
+from toil.options.common import SYS_MAX_SIZE
 from toil.job import JobDescription, Requirer
 from toil.lib.conversions import human2bytes
 from toil.lib.misc import get_user_name, slow_down, utc_now

src/toil/batchSystems/options.py

@@ -80,7 +80,7 @@ def set_batchsystem_options(batch_system: Optional[str], set_option: OptionSette
 
 
 def add_all_batchsystem_options(parser: Union[ArgumentParser, _ArgumentGroup]) -> None:
-    from toil.common import SYS_MAX_SIZE
+    from toil.options.common import SYS_MAX_SIZE
 
     # Do the global cross-batch-system arguments
     parser.add_argument(

src/toil/batchSystems/singleMachine.py

@@ -34,7 +34,8 @@
                                                    UpdatedBatchJobInfo)
 from toil.batchSystems.options import OptionSetter
 from toil.bus import ExternalBatchIdMessage
-from toil.common import SYS_MAX_SIZE, Config, Toil, make_open_interval_action
+from toil.common import Config, Toil
+from toil.options.common import SYS_MAX_SIZE, make_open_interval_action
 from toil.job import (AcceleratorRequirement,
                       JobDescription,
                       Requirer,