Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Anonymous AWS Access #5205

Draft
wants to merge 11 commits into
base: master
Choose a base branch
from
2 changes: 2 additions & 0 deletions docs/appendices/environment_vars.rst
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ There are several environment variables that affect the way Toil runs.
| | access to S3 and SimpleDB, the AWS job store will |
| | not be usable. |
+----------------------------------+----------------------------------------------------+
| TOIL_AWS_ANONYMOUS_URL_ACCESS | Whether to access s3:// URLs anonymously. |
+----------------------------------+----------------------------------------------------+
| TOIL_GOOGLE_PROJECTID | The Google project ID to use when generating |
| | Google job store names for tests or CWL workflows. |
+----------------------------------+----------------------------------------------------+
Expand Down
4 changes: 4 additions & 0 deletions docs/running/cliOptions.rst
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,10 @@ Allows configuring Toil's data storage.
of all the jobs reading from it at once, and you want
to use ``--caching=True`` to make jobs on each node
read from node-local cache storage. (Default=True)
--awsAnonymousUrlAccess BOOL
Whether to access AWS S3 URLs anonymously. Useful for
skipping multi-factor authentication when MFA is
configured but unnecessary.

**Autoscaling Options**
Allows the specification of the minimum and maximum number of nodes in an
Expand Down
9 changes: 5 additions & 4 deletions src/toil/batchSystems/abstractBatchSystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
from threading import Condition
from typing import Any, ContextManager, NamedTuple, Optional, Union, cast

from toil.batchSystems.options import OptionSetter
from toil.options import OptionSetter
from toil.bus import MessageBus, MessageOutbox
from toil.common import Config, Toil, cacheDirName
from toil.deferred import DeferredFunctionManager
Expand Down Expand Up @@ -282,16 +282,17 @@ def add_options(cls, parser: Union[ArgumentParser, _ArgumentGroup]) -> None:
"""
If this batch system provides any command line options, add them to the given parser.
"""
pass

@classmethod
def setOptions(cls, setOption: OptionSetter) -> None:
"""
Process command line or configuration options relevant to this batch system.

:param setOption: A function with signature
setOption(option_name, parsing_function=None, check_function=None, default=None, env=None)
returning nothing, used to update run configuration as a side effect.
:param setOption: A function taking an option name and returning
nothing, used to update run configuration as a side effect.
"""
pass

def getWorkerContexts(self) -> list[ContextManager[Any]]:
"""
Expand Down
2 changes: 1 addition & 1 deletion src/toil/batchSystems/awsBatch.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
)
from toil.batchSystems.cleanup_support import BatchSystemCleanupSupport
from toil.batchSystems.contained_executor import pack_job
from toil.batchSystems.options import OptionSetter
from toil.options import OptionSetter
from toil.bus import ExternalBatchIdMessage
from toil.common import Config, Toil
from toil.job import JobDescription, Requirer
Expand Down
2 changes: 1 addition & 1 deletion src/toil/batchSystems/kubernetes.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@
)
from toil.batchSystems.cleanup_support import BatchSystemCleanupSupport
from toil.batchSystems.contained_executor import pack_job
from toil.batchSystems.options import OptionSetter
from toil.options import OptionSetter
from toil.common import Config, Toil
from toil.job import JobDescription, Requirer
from toil.lib.conversions import human2bytes
Expand Down
2 changes: 1 addition & 1 deletion src/toil/batchSystems/mesos/batchSystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
)
from toil.batchSystems.local_support import BatchSystemLocalSupport
from toil.batchSystems.mesos import JobQueue, MesosShape, TaskData, ToilJob
from toil.batchSystems.options import OptionSetter
from toil.options import OptionSetter
from toil.job import JobDescription
from toil.lib.conversions import b_to_mib, mib_to_b
from toil.lib.memoize import strict_bool
Expand Down
30 changes: 5 additions & 25 deletions src/toil/batchSystems/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,40 +13,20 @@

import logging
from argparse import ArgumentParser, _ArgumentGroup
from typing import Any, Callable, Optional, Protocol, TypeVar, Union
from typing import Optional, Union

from toil.batchSystems.registry import (
DEFAULT_BATCH_SYSTEM,
get_batch_system,
get_batch_systems,
)
from toil.lib.threading import cpu_count
# Need to re-export from here for compatibility with old batch system plugins
from toil.options import OptionSetter as OptionSetter

logger = logging.getLogger(__name__)


class OptionSetter(Protocol):
"""
Protocol for the setOption function we get to let us set up CLI options for
each batch system.

Actual functionality is defined in the Config class.
"""

OptionType = TypeVar("OptionType")

def __call__(
self,
option_name: str,
parsing_function: Optional[Callable[[Any], OptionType]] = None,
check_function: Optional[Callable[[OptionType], Union[None, bool]]] = None,
default: Optional[OptionType] = None,
env: Optional[list[str]] = None,
old_names: Optional[list[str]] = None,
) -> bool: ...


def set_batchsystem_options(
def set_batch_system_options(
batch_system: Optional[str], set_option: OptionSetter
) -> None:
"""
Expand Down Expand Up @@ -80,7 +60,7 @@ def set_batchsystem_options(
set_option("batch_logs_dir")


def add_all_batchsystem_options(parser: Union[ArgumentParser, _ArgumentGroup]) -> None:
def add_all_batch_system_options(parser: Union[ArgumentParser, _ArgumentGroup]) -> None:
from toil.options.common import SYS_MAX_SIZE

# Do the global cross-batch-system arguments
Expand Down
2 changes: 1 addition & 1 deletion src/toil/batchSystems/singleMachine.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
ResourceSet,
UpdatedBatchJobInfo,
)
from toil.batchSystems.options import OptionSetter
from toil.options import OptionSetter
from toil.bus import ExternalBatchIdMessage
from toil.common import Config, Toil
from toil.job import (
Expand Down
2 changes: 1 addition & 1 deletion src/toil/batchSystems/slurm.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
from toil.batchSystems.abstractGridEngineBatchSystem import (
AbstractGridEngineBatchSystem,
)
from toil.batchSystems.options import OptionSetter
from toil.options import OptionSetter
from toil.bus import get_job_kind
from toil.common import Config
from toil.job import JobDescription, Requirer
Expand Down
Loading
Loading