Skip to content

Commit f632e46

Browse files
johanneskoesterjohanneskoester
committed
cleanup, WIP
1 parent ee011a3 commit f632e46

File tree

1 file changed

+2
-151
lines changed

1 file changed

+2
-151
lines changed

‎README.md

Lines changed: 2 additions & 151 deletions
Original file line numberDiff line numberDiff line change
@@ -1,152 +1,3 @@
1-
# Stable interfaces and functionality for Snakemake executor plugins
1+
# Stable interfaces and functionality for Snakemake software deployment plugins
22

3-
This package provides a stable interface for interactions between Snakemake and its executor plugins (WIP).
4-
5-
Plugins should implement the following skeleton to comply with this interface.
6-
It is recommended to use Snakemake's poetry plugin to set up this skeleton (and automated testing) within a python package, see https://github.com/snakemake/poetry-snakemake-plugin.
7-
8-
```python
9-
from dataclasses import dataclass, field
10-
from typing import List, Generator, Optional
11-
from snakemake_interface_executor_plugins.executors.base import SubmittedJobInfo
12-
from snakemake_interface_executor_plugins.executors.remote import RemoteExecutor
13-
from snakemake_interface_executor_plugins.settings import (
14-
ExecutorSettingsBase, CommonSettings
15-
)
16-
from snakemake_interface_executor_plugins.workflow import WorkflowExecutorInterface
17-
from snakemake_interface_executor_plugins.logging import LoggerExecutorInterface
18-
from snakemake_interface_executor_plugins.jobs import (
19-
JobExecutorInterface,
20-
)
21-
22-
# Optional:
23-
# Define additional settings for your executor.
24-
# They will occur in the Snakemake CLI as --<executor-name>-<param-name>
25-
# Omit this class if you don't need any.
26-
# Make sure that all defined fields are Optional and specify a default value
27-
# of None or anything else that makes sense in your case.
28-
@dataclass
29-
class ExecutorSettings(ExecutorSettingsBase):
30-
myparam: Optional[int] = field(
31-
default=None,
32-
metadata={
33-
"help": "Some help text",
34-
# Optionally request that setting is also available for specification
35-
# via an environment variable. The variable will be named automatically as
36-
# SNAKEMAKE_<executor-name>_<param-name>, all upper case.
37-
# This mechanism should only be used for passwords and usernames.
38-
# For other items, we rather recommend to let people use a profile
39-
# for setting defaults
40-
# (https://snakemake.readthedocs.io/en/stable/executing/cli.html#profiles).
41-
"env_var": False,
42-
# Optionally specify a function that parses the value given by the user.
43-
# This is useful to create complex types from the user input.
44-
"parse_func": ...,
45-
# If a parse_func is specified, you also have to specify an unparse_func
46-
# that converts the parsed value back to a string.
47-
"unparse_func": ...,
48-
# Optionally specify that setting is required when the executor is in use.
49-
"required": True,
50-
# Optionally specify multiple args with "nargs": True
51-
},
52-
)
53-
54-
55-
# Required:
56-
# Specify common settings shared by various executors.
57-
common_settings = CommonSettings(
58-
# define whether your executor plugin executes locally
59-
# or remotely. In virtually all cases, it will be remote execution
60-
# (cluster, cloud, etc.). Only Snakemake's standard execution
61-
# plugins (snakemake-executor-plugin-dryrun, snakemake-executor-plugin-local)
62-
# are expected to specify False here.
63-
non_local_exec=True,
64-
# Whether the executor implies to not have a shared file system
65-
implies_no_shared_fs=True,
66-
# whether to deploy workflow sources to default storage provider before execution
67-
job_deploy_sources=True,
68-
# whether arguments for setting the storage provider shall be passed to jobs
69-
pass_default_storage_provider_args=True,
70-
# whether arguments for setting default resources shall be passed to jobs
71-
pass_default_resources_args=True,
72-
# whether environment variables shall be passed to jobs (if False, use
73-
# self.envvars() to obtain a dict of environment variables and their values
74-
# and pass them e.g. as secrets to the execution backend)
75-
pass_envvar_declarations_to_cmd=True,
76-
# whether the default storage provider shall be deployed before the job is run on
77-
# the remote node. Usually set to True if the executor does not assume a shared fs
78-
auto_deploy_default_storage_provider=True,
79-
# specify initial amount of seconds to sleep before checking for job status
80-
init_seconds_before_status_checks=0,
81-
)
82-
83-
84-
# Required:
85-
# Implementation of your executor
86-
class Executor(RemoteExecutor):
87-
def __post_init__(self):
88-
# access workflow
89-
self.workflow
90-
# access executor specific settings
91-
self.workflow.executor_settings
92-
93-
# IMPORTANT: in your plugin, only access methods and properties of
94-
# Snakemake objects (like Workflow, Persistence, etc.) that are
95-
# defined in the interfaces found in the
96-
# snakemake-interface-executor-plugins and the
97-
# snakemake-interface-common package.
98-
# Other parts of those objects are NOT guaranteed to remain
99-
# stable across new releases.
100-
101-
# To ensure that the used interfaces are not changing, you should
102-
# depend on these packages as >=a.b.c,<d with d=a+1 (i.e. pin the
103-
# dependency on this package to be at least the version at time
104-
# of development and less than the next major version which would
105-
# introduce breaking changes).
106-
107-
# In case of errors outside of jobs, please raise a WorkflowError
108-
109-
def run_job(self, job: JobExecutorInterface):
110-
# Implement here how to run a job.
111-
# You can access the job's resources, etc.
112-
# via the job object.
113-
# After submitting the job, you have to call
114-
# self.report_job_submission(job_info).
115-
# with job_info being of type
116-
# snakemake_interface_executor_plugins.executors.base.SubmittedJobInfo.
117-
# If required, make sure to pass the job's id to the job_info object, as keyword
118-
# argument 'external_job_id'.
119-
120-
...
121-
122-
async def check_active_jobs(
123-
self, active_jobs: List[SubmittedJobInfo]
124-
) -> Generator[SubmittedJobInfo, None, None]:
125-
# Check the status of active jobs.
126-
127-
# You have to iterate over the given list active_jobs.
128-
# If you provided it above, each will have its external_jobid set according
129-
# to the information you provided at submission time.
130-
# For jobs that have finished successfully, you have to call
131-
# self.report_job_success(active_job).
132-
# For jobs that have errored, you have to call
133-
# self.report_job_error(active_job).
134-
# This will also take care of providing a proper error message.
135-
# Usually there is no need to perform additional logging here.
136-
# Jobs that are still running have to be yielded.
137-
#
138-
# For queries to the remote middleware, please use
139-
# self.status_rate_limiter like this:
140-
#
141-
# async with self.status_rate_limiter:
142-
# # query remote middleware here
143-
#
144-
# To modify the time until the next call of this method,
145-
# you can set self.next_sleep_seconds here.
146-
...
147-
148-
def cancel_jobs(self, active_jobs: List[SubmittedJobInfo]):
149-
# Cancel all active jobs.
150-
# This method is called when Snakemake is interrupted.
151-
...
152-
```
3+
This package provides a stable interface for interactions between Snakemake and its software deployment plugins.

0 commit comments

Comments
 (0)